{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T03:47:05Z","timestamp":1776052025247,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":32,"publisher":"ACM","license":[{"start":{"date-parts":[[2027,4,13]],"date-time":"2027-04-13T00:00:00Z","timestamp":1807574400000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Air Force office of scientific research","award":["FA9550-23-1-0453"],"award-info":[{"award-number":["FA9550-23-1-0453"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,13]]},"DOI":"10.1145\/3772363.3799244","type":"proceedings-article","created":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T01:55:24Z","timestamp":1776045324000},"page":"1-28","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Mapping the Spectrum of Automatability: A Framework for LLM-Assisted Annotation of Team Communication and Metacognition"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8353-8989","authenticated-orcid":false,"given":"Senjuti","family":"Dutta","sequence":"first","affiliation":[{"name":"Institute of Cognitive Science, University of Colorado Boulder, Boulder, Colorado, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2751-0874","authenticated-orcid":false,"given":"Emily","family":"Doherty","sequence":"additional","affiliation":[{"name":"Institute of Cognitive Science, University of Colorado Boulder, Boulder, Colorado, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0111-6948","authenticated-orcid":false,"given":"Leanne","family":"Hirshfield","sequence":"additional","affiliation":[{"name":"Institute of Cognitive Science, University of Colorado, Boulder, Colorado, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,4,13]]},"reference":[{"key":"e_1_3_3_1_2_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544549.3582749"},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"crossref","unstructured":"William\u00a0G Cochran. 1950. The comparison of percentages in matched samples. Biometrika 37 3\/4 (1950) 256\u2013266.","DOI":"10.1093\/biomet\/37.3-4.256"},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"crossref","unstructured":"Nancy\u00a0J Cooke Jamie\u00a0C Gorman and Preston\u00a0A Kiekel. 2017. Communication as team-level cognitive processing. Macrocognition in teams (2017).","DOI":"10.1201\/9781315593166-4"},{"key":"e_1_3_3_1_5_2","unstructured":"Shih-Chieh Dai Aiping Xiong and Lun-Wei Ku. 2023. LLM-in-the-loop: Leveraging large language model for thematic analysis. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.15100 (2023)."},{"key":"e_1_3_3_1_6_2","doi-asserted-by":"crossref","unstructured":"Olive\u00a0Jean Dunn. 1961. Multiple comparisons among means. Journal of the American statistical association 56 293 (1961) 52\u201364.","DOI":"10.1080\/01621459.1961.10482090"},{"key":"e_1_3_3_1_7_2","volume-title":"Team Effectiveness in Complex Organizations and Systems: Cross-disciplinary perspectives and approaches","author":"Foltz Peter\u00a0W","year":"2008","unstructured":"Peter\u00a0W Foltz and Melanie\u00a0J Martin. 2008. Automated Communication Analysis of Teams. In Team Effectiveness in Complex Organizations and Systems: Cross-disciplinary perspectives and approaches, E. Salas, G. F. Goodwin, & S. Burke (Ed.). Routledge, New York."},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"crossref","unstructured":"Milton Friedman. 1937. The use of ranks to avoid the assumption of normality implicit in the analysis of variance. Journal of the american statistical association 32 200 (1937) 675\u2013701.","DOI":"10.1080\/01621459.1937.10503522"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"crossref","unstructured":"Fabrizio Gilardi Meysam Alizadeh and Ma\u00ebl Kubli. 2023. ChatGPT outperforms crowd workers for text-annotation tasks. Proceedings of the National Academy of Sciences 120 30 (2023) e2305016120.","DOI":"10.1073\/pnas.2305016120"},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.naacl-industry.15"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3543873.3587368"},{"key":"e_1_3_3_1_12_2","unstructured":"Taja Kuzman Igor Mozeti\u010d and Nikola Ljube\u0161i\u0107. 2023. Chatgpt: Beginning of an end of manual linguistic data annotation? use case of automatic genre identification. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.03953 (2023)."},{"key":"e_1_3_3_1_13_2","doi-asserted-by":"publisher","unstructured":"Quinn McNemar. 1947. Note on the sampling error of the difference between correlated proportions or percentages. Psychometrika 12 2 (1947) 153\u2013157. 10.1007\/bf02295996","DOI":"10.1007\/bf02295996"},{"key":"e_1_3_3_1_14_2","unstructured":"Anmol\u00a0Girish More. 2024. Analyzing Team Processes Through Team Communication Using Large Language Models. Master\u2019s thesis. Arizona State University."},{"key":"e_1_3_3_1_15_2","unstructured":"OpenAI. 2024. Hello GPT-4o. https:\/\/openai.com\/index\/hello-gpt-4o\/ Accessed: 2025-12-31."},{"key":"e_1_3_3_1_16_2","unstructured":"Jason\u00a0G Reitman. 2022. Generalizable Communication Styles in Novice and Expert Team Performance. Ph.\u00a0D. Dissertation. University of California Irvine."},{"key":"e_1_3_3_1_17_2","unstructured":"Sneha Saj. 2024. Exploring the Utility of Large Language Models for Producing Insights from Discussion Recordings. (2024)."},{"key":"e_1_3_3_1_18_2","unstructured":"Neda\u00a0Taghizadeh Serajeh Iman Mohammadi Vittorio Fuccella and Mattia De\u00a0Rosa. 2024. LLMs in HCI Data Work: Bridging the Gap Between Information Retrieval and Responsible Research Practices. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2403.18173 (2024)."},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"crossref","unstructured":"Thomas\u00a0B Sheridan and William\u00a0L Verplank. 1978. Human and computer control of undersea teleoperators. (1978).","DOI":"10.21236\/ADA057655"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"crossref","unstructured":"Randall Spain Wookhee Min Vikram Kumaran Jay Pande Jason Saville and James Lester. 2025. Applying Large Language Models to Enhance Dialogue and Communication Analysis for Adaptive Team Training. International Journal of Artificial Intelligence in Education (2025) 1\u201335.","DOI":"10.1007\/s40593-025-00479-5"},{"key":"e_1_3_3_1_21_2","unstructured":"Petter T\u00f6rnberg. 2023. Chatgpt-4 outperforms experts and crowd workers in annotating political twitter messages with zero-shot learning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2304.06588 (2023)."},{"key":"e_1_3_3_1_22_2","unstructured":"Petter T\u00f6rnberg. 2024. Best practices for text annotation with large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.05129 (2024)."},{"key":"e_1_3_3_1_23_2","doi-asserted-by":"crossref","unstructured":"Marialena Vagia Aksel\u00a0A Transeth and Sigurd\u00a0A Fjerdingen. 2016. A literature review on the levels of automation during the years. What are the different taxonomies that have been proposed? Applied ergonomics 53 (2016) 190\u2013202.","DOI":"10.1016\/j.apergo.2015.09.013"},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"crossref","unstructured":"Qile Wang Moath Erqsous Kenneth\u00a0E Barner and Matthew\u00a0Louis Mauriello. 2025. LATA: A Pilot Study on LLM-Assisted Thematic Analysis of Online Social Network Data Generation Experiences. Proceedings of the ACM on Human-Computer Interaction 9 2 (2025) 1\u201328.","DOI":"10.1145\/3711022"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641960"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"crossref","unstructured":"Matthijs\u00a0J Warrens. 2011. Cohen\u2019s kappa is a weighted average. Statistical Methodology 8 6 (2011) 473\u2013484.","DOI":"10.1016\/j.stamet.2011.06.002"},{"key":"e_1_3_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581754.3584136"},{"key":"e_1_3_3_1_28_2","unstructured":"Eddie Yang Zoey Wang Carl Zhou and Yaosheng Xu. 2025. Data Annotation with Large Language Models: Lessons from a Large Empirical Evaluation. (2025)."},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"crossref","unstructured":"Cristina\u00a0D Zepeda and Timothy\u00a0J Nokes-Malach. 2023. Assessing metacognitive regulation during problem solving: A comparison of three measures. Journal of Intelligence 11 1 (2023) 16.","DOI":"10.3390\/jintelligence11010016"},{"key":"e_1_3_3_1_30_2","unstructured":"He Zhang Chuhao Wu Jingyi Xie Fiona Rubino Sydney Graver ChanMin Kim John\u00a0M Carroll and Jie Cai. 2024. When qualitative research meets large language model: Exploring the potential of QualiGPT as a tool for qualitative coding. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.14925 (2024)."},{"key":"e_1_3_3_1_31_2","doi-asserted-by":"publisher","DOI":"10.35542\/osf.io\/hrtz6"},{"key":"e_1_3_3_1_32_2","unstructured":"Yiming Zhu Peixian Zhang Ehsan-Ul Haq Pan Hui and Gareth Tyson. 2023. Can chatgpt reproduce human-generated labels? a study of social computing tasks. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2304.10145 (2023)."},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"crossref","unstructured":"Caleb Ziems William Held Omar Shaikh Jiaao Chen Zhehao Zhang and Diyi Yang. 2024. Can large language models transform computational social science? Computational Linguistics 50 1 (2024) 237\u2013291.","DOI":"10.1162\/coli_a_00502"}],"event":{"name":"CHI EA '26: Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems","location":"Barcelona , Spain","acronym":"CHI EA '26","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3772363.3799244","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3772363.3799244","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T03:16:01Z","timestamp":1776050161000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3772363.3799244"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,13]]},"references-count":32,"alternative-id":["10.1145\/3772363.3799244","10.1145\/3772363"],"URL":"https:\/\/doi.org\/10.1145\/3772363.3799244","relation":{},"subject":[],"published":{"date-parts":[[2026,4,13]]},"assertion":[{"value":"2026-04-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}