{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T14:29:34Z","timestamp":1776090574465,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":63,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,3,18]],"date-time":"2024-03-18T00:00:00Z","timestamp":1710720000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,3,18]]},"DOI":"10.1145\/3640543.3645214","type":"proceedings-article","created":{"date-parts":[[2024,4,5]],"date-time":"2024-04-05T18:23:12Z","timestamp":1712341392000},"page":"405-418","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":9,"title":["Closing the Knowledge Gap in Designing Data Annotation Interfaces for AI-powered Disaster Management Analytic Systems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-2479-8846","authenticated-orcid":false,"given":"Zinat","family":"Ara","sequence":"first","affiliation":[{"name":"Information Sciences and Technology, George Mason University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3462-9037","authenticated-orcid":false,"given":"Hossein","family":"Salemi","sequence":"additional","affiliation":[{"name":"Information Sciences and Technology, George Mason University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6050-5404","authenticated-orcid":false,"given":"Sungsoo Ray","family":"Hong","sequence":"additional","affiliation":[{"name":"Information Sciences and Technology, George Mason University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8759-378X","authenticated-orcid":false,"given":"Yasas","family":"Senarath","sequence":"additional","affiliation":[{"name":"Information Sciences and Technology, George Mason University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5429-3855","authenticated-orcid":false,"given":"Steve","family":"Peterson","sequence":"additional","affiliation":[{"name":"Montgomery County Community Emergency Response Team, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7506-3343","authenticated-orcid":false,"given":"Amanda Lee","family":"Hughes","sequence":"additional","affiliation":[{"name":"Computer Science Department, Brigham Young University, United States"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4573-8450","authenticated-orcid":false,"given":"Hemant","family":"Purohit","sequence":"additional","affiliation":[{"name":"Information Sciences and Technology, George Mason University, United States"}]}],"member":"320","published-online":{"date-parts":[[2024,4,5]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"The intellectual challenge of CSCW: the gap between social requirements and technical feasibility. Human\u2013Computer Interaction 15, 2-3","author":"Ackerman S","year":"2000","unstructured":"Mark\u00a0S Ackerman. 2000. The intellectual challenge of CSCW: the gap between social requirements and technical feasibility. Human\u2013Computer Interaction 15, 2-3 (2000), 179\u2013203."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357311"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.18293\/SEKE2021-009"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData52589.2021.9671381"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449163"},{"key":"e_1_3_2_1_6_1","volume-title":"Transactive memory in organizational groups: the effects of content, consensus, specialization, and accuracy on group performance.Journal of applied psychology 88, 5","author":"Austin R","year":"2003","unstructured":"John\u00a0R Austin. 2003. Transactive memory in organizational groups: the effects of content, consensus, specialization, and accuracy on group performance.Journal of applied psychology 88, 5 (2003), 866."},{"key":"e_1_3_2_1_7_1","unstructured":"Yuntao Bai Andy Jones Kamal Ndousse Amanda Askell Anna Chen Nova DasSarma Dawn Drain Stanislav Fort Deep Ganguli Tom Henighan Nicholas Joseph Saurav Kadavath Jackson Kernion Tom Conerly Sheer El-Showk Nelson Elhage Zac Hatfield-Dodds Danny Hernandez Tristan Hume Scott Johnston Shauna Kravec Liane Lovitt Neel Nanda Catherine Olsson Dario Amodei Tom Brown Jack Clark Sam McCandlish Chris Olah Ben Mann and Jared Kaplan. 2022. Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback. arxiv:2204.05862\u00a0[cs.CL]"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00338"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998255"},{"key":"e_1_3_2_1_10_1","first-page":"31","article-title":"Normalized (pointwise) mutual information in collocation extraction","volume":"30","author":"Bouma Gerlof","year":"2009","unstructured":"Gerlof Bouma. 2009. Normalized (pointwise) mutual information in collocation extraction. Proceedings of GSCL 30 (2009), 31\u201340.","journal-title":"Proceedings of GSCL"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555212"},{"key":"e_1_3_2_1_12_1","article-title":"Complete counterbalancing of immediate sequential effects in a Latin square design","volume":"53","author":"Bradley V","year":"1958","unstructured":"James\u00a0V Bradley. 1958. Complete counterbalancing of immediate sequential effects in a Latin square design. J. Amer. Statist. Assoc. 53, 282 (1958). https:\/\/www.tandfonline.com\/doi\/abs\/10.1080\/01621459.1958.10501456","journal-title":"J. Amer. Statist. Assoc."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1080\/09585190500404614"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11069-020-04133-2"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1080\/01431160802672864"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300522"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476076"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300460"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the ACM on Human-Computer Interaction 3, CSCW","author":"Joon\u00a0Young Chung John","year":"2019","unstructured":"John Joon\u00a0Young Chung, Jean\u00a0Y Song, Sindhu Kutty, Sungsoo Hong, Juho Kim, and Walter\u00a0S Lasecki. 2019. Efficient elicitation approaches to estimate collective crowd answers. Proceedings of the ACM on Human-Computer Interaction 3, CSCW (2019), 1\u201325."},{"key":"e_1_3_2_1_21_1","unstructured":"U.S. Department of Homeland\u00a0Security FEMA. 2022. Community Emergency Response Team (CERT). https:\/\/www.ready.gov\/cert April 19."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM51629.2021.00023"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539419"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3555590"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.law-1.18"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173640"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359208"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3392878"},{"key":"e_1_3_2_1_29_1","volume-title":"CHI 2019 Workshop, Emerging Perspectives in Human-Centered Machine Learning. ACM.","author":"Hong Sungsoo\u00a0Ray","year":"2019","unstructured":"Sungsoo\u00a0Ray Hong, Jorge\u00a0Piazentin Ono, Juliana Freire, and Enrico Bertini. 2019. Disseminating Machine Learning to domain experts: Understanding challenges and opportunities in supporting a model building process. In CHI 2019 Workshop, Emerging Perspectives in Human-Centered Machine Learning. ACM."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173640"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359208"},{"key":"e_1_3_2_1_32_1","unstructured":"Amanda\u00a0L. Hughes Steve Peterson and Leysia Palen. 2014. Social media in emergency management. 349\u2013392 pages."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i11.21521"},{"key":"e_1_3_2_1_34_1","unstructured":"OpenAI Inc.2023. ChatGPT. https:\/\/chat.openai.com\/ Accessed: 2023-09-20."},{"key":"e_1_3_2_1_35_1","unstructured":"Twitter Inc.2023. Twitter. https:\/\/twitter.com\/ Accessed: 2023-09-13."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2818048.2820016"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580645"},{"key":"e_1_3_2_1_38_1","volume-title":"Transformers in vision: A survey. ACM computing surveys (CSUR) 54, 10s","author":"Khan Salman","year":"2022","unstructured":"Salman Khan, Muzammal Naseer, Munawar Hayat, Syed\u00a0Waqas Zamir, Fahad\u00a0Shahbaz Khan, and Mubarak Shah. 2022. Transformers in vision: A survey. ACM computing surveys (CSUR) 54, 10s (2022), 1\u201341."},{"key":"e_1_3_2_1_39_1","volume-title":"Research Methods in Human-Computer Interaction","author":"Lazar Jonathan","unstructured":"Jonathan Lazar, Jinjuan\u00a0Heidi Feng, and Harry Hochheiser. 2010. Research Methods in Human-Computer Interaction. Wiley Publishing."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00622"},{"key":"e_1_3_2_1_41_1","volume-title":"An integrated iterative annotation technique for easing neural network training in medical image analysis. Nature machine intelligence 1, 2","author":"Lutnick Brendon","year":"2019","unstructured":"Brendon Lutnick, Brandon Ginley, Darshana Govind, Sean\u00a0D McGarry, Peter\u00a0S LaViolette, Rabi Yacoub, Sanjay Jain, John\u00a0E Tomaszewski, Kuang-Yu Jen, and Pinaki Sarder. 2019. An integrated iterative annotation technique for easing neural network training in medical image analysis. Nature machine intelligence 1, 2 (2019), 112\u2013119."},{"key":"e_1_3_2_1_42_1","volume-title":"2020 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). IEEE, 1226\u20131233","author":"Madono Koki","year":"2020","unstructured":"Koki Madono, Teppei Nakano, Tetsunori Kobayashi, and Tetsuji Ogawa. 2020. Efficient human-in-the-loop object detection using bi-directional deep sort and annotation-free segment identification. In 2020 Asia-Pacific Signal and Information Processing Association Annual Summit and Conference (APSIPA ASC). IEEE, 1226\u20131233."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.5555\/1543767.1543770"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2022.102772"},{"key":"e_1_3_2_1_45_1","unstructured":"Steve Peterson Keri Stephens Amanda Hughes and Hemant Purohit. 2019. When Official Systems Overload: A Framework for Finding Social Media Calls for Help during Evacuations. (2019) 867\u2013875."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1002\/acp.3676"},{"key":"e_1_3_2_1_47_1","volume-title":"Looking Back: Retrospective Study Methods for HCI. In Ways of Knowing in HCI. https:\/\/api.semanticscholar.org\/CorpusID:2470741","author":"Russell M.","year":"2014","unstructured":"Daniel\u00a0M. Russell and Ed\u00a0H. Chi. 2014. Looking Back: Retrospective Study Methods for HCI. In Ways of Knowing in HCI. https:\/\/api.semanticscholar.org\/CorpusID:2470741"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.watres.2021.117695"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328519.3329134"},{"key":"e_1_3_2_1_50_1","volume-title":"International Handbook of Disaster Research","author":"Senarath Yasas","unstructured":"Yasas Senarath, Rahul Pandey, Steve Peterson, and Hemant Purohit. 2022. Citizen-Helper System for Human-Centered AI Use in Disaster Management. In International Handbook of Disaster Research. Springer, 1\u201321."},{"key":"e_1_3_2_1_51_1","volume-title":"Proceedings of the 2021 international conference on social computing, behavioral-cultural modeling & prediction and behavior representation in modeling and simulation.","author":"Senarath Yasas","year":"2021","unstructured":"Yasas Senarath, Steve Peterson, Hemant Purohit, Amanda\u00a0L Hughes, and Keri\u00a0K Stephens. 2021. Mining risk behaviors from social media for pandemic crisis preparedness and response. In Proceedings of the 2021 international conference on social computing, behavioral-cultural modeling & prediction and behavior representation in modeling and simulation."},{"key":"e_1_3_2_1_52_1","volume-title":"Knowledge Engineering and Knowledge Management: Third International Joint Conference, IC3K 2011","author":"Serrano Mart\u00edn","year":"2013","unstructured":"Mart\u00edn Serrano, Lei Shi, M\u00edche\u00e1l\u00a0\u00d3 Foghl\u00fa, and William Donnelly. 2013. Cloud services composition support by using semantic annotation and linked data. In Knowledge Discovery, Knowledge Engineering and Knowledge Management: Third International Joint Conference, IC3K 2011, Paris, France, October 26-29, 2011. Revised Selected Papers 3. Springer, 278\u2013293."},{"key":"e_1_3_2_1_53_1","unstructured":"National\u00a0Weather Service. 2023. Hurricane IAN. https:\/\/www.weather.gov\/mhx\/HurricaneIan093022 Accessed: 2023-09-13."},{"key":"e_1_3_2_1_54_1","volume-title":"International Conference on Intelligent Technologies and Applications. Springer, 287\u2013299","author":"Shahi Gautam\u00a0Kishore","year":"2021","unstructured":"Gautam\u00a0Kishore Shahi and Tim\u00a0A Majchrzak. 2021. Amused: an annotation framework of multimodal social media data. In International Conference on Intelligent Technologies and Applications. Springer, 287\u2013299."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/IV.2000.859742"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581431"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3610187"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/1082983.1083119"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13293"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","unstructured":"Ding Wang Shantanu Prabhat and Nithya Sambasivan. 2022. Whose AI Dream? In search of the aspiration in data annotation. arxiv:2203.10748\u00a0[cs.HC]","DOI":"10.1145\/3491102.3502121"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3512940"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502075"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.968"}],"event":{"name":"IUI '24: 29th International Conference on Intelligent User Interfaces","location":"Greenville SC USA","acronym":"IUI '24","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 29th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3640543.3645214","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3640543.3645214","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T00:56:47Z","timestamp":1764550607000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3640543.3645214"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,18]]},"references-count":63,"alternative-id":["10.1145\/3640543.3645214","10.1145\/3640543"],"URL":"https:\/\/doi.org\/10.1145\/3640543.3645214","relation":{},"subject":[],"published":{"date-parts":[[2024,3,18]]},"assertion":[{"value":"2024-04-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}