{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,2]],"date-time":"2026-10-02T19:30:54Z","timestamp":1790969454265,"version":"4.1.0"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,5,6]],"date-time":"2021-05-06T00:00:00Z","timestamp":1620259200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,5,6]]},"DOI":"10.1145\/3411764.3445522","type":"proceedings-article","created":{"date-parts":[[2021,5,7]],"date-time":"2021-05-07T22:27:22Z","timestamp":1620426442000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":65,"title":["Assessing the Impact of Automated Suggestions on Decision Making: Domain Experts Mediate Model Errors but Take Less Initiative"],"prefix":"10.1145","author":[{"given":"Ariel","family":"Levy","sequence":"first","affiliation":[{"name":"CSAIL MIT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Monica","family":"Agrawal","sequence":"additional","affiliation":[{"name":"CSAIL MIT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arvind","family":"Satyanarayan","sequence":"additional","affiliation":[{"name":"CSAIL MIT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David","family":"Sontag","sequence":"additional","affiliation":[{"name":"CSAIL MIT, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,5,7]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Robust Benchmarking for Clinical Entity Recognition and Normalization. Conference on Machine Learning for Healthcare (MLHC)","author":"Agrawal Monica","year":"2020","unstructured":"Monica Agrawal, Chloe O\u2019Connell, Yasmin Fatemi, Ariel Levy, and David Sontag. 2020. Robust Benchmarking for Clinical Entity Recognition and Normalization. Conference on Machine Learning for Healthcare (MLHC) (2020)."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1136\/jamia.2009.002733"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346431"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377498"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","unstructured":"Carrie\u00a0J. Cai Samantha Winter David Steiner Lauren Wilcox and Michael Terry. 2019. \u201cHello Ai\u201d: Uncovering the onboarding needs of medical practitioners for human\u2013AI collaborative decision-making. https:\/\/doi.org\/10.1145\/3359206","DOI":"10.1145\/3359206"},{"key":"e_1_3_2_2_7_1","volume-title":"Shared mental models in expert team decision making","author":"Cannon-Bowers J\u00a0A","year":"1993","unstructured":"J\u00a0A Cannon-Bowers, E Salas, and S Converse. 1993. Shared mental models in expert team decision making. Individual and Group Decision Making(1993)."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/s15-2051"},{"key":"e_1_3_2_2_9_1","volume-title":"ACL 2010 - LAW 2010: 4th Linguistic Annotation Workshop, Proceedings.","author":"Fort Kar\u00ebn","year":"2010","unstructured":"Kar\u00ebn Fort and Beno\u00eet Sagot. 2010. Influence of pre-annotation on POS-tagged corpus development. In ACL 2010 - LAW 2010: 4th Linguistic Annotation Workshop, Proceedings."},{"key":"e_1_3_2_2_10_1","volume-title":"Medlee-a medical language extraction and encoding system","author":"Friedman Carol","year":"1995","unstructured":"Carol Friedman. 1995. Medlee-a medical language extraction and encoding system. Columbia University, and Queens College of CUNY (1995)."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.3115\/1642059.1642068"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","unstructured":"Katy\u00a0Ilonka Gero Zahra Ashktorab Casey Dugan Qian Pan James Johnson Werner Geyer Maria Ruiz Sarah Miller David\u00a0R. Millen Murray Campbell Sadhana Kumaravel and Wei Zhang. 2020. Mental Models of AI Agents in a Cooperative Game Setting. https:\/\/doi.org\/10.1145\/3313831.3376316","DOI":"10.1145\/3313831.3376316"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2013-002255"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2012.324"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1130"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-015-9204-8_10"},{"key":"e_1_3_2_2_17_1","volume-title":"Annual Symposium on Computer Application [sic] in Medical Care. Symposium on Computer Applications in Medical Care","author":"Harbourt M.","year":"1993","unstructured":"A.\u00a0M. Harbourt, E.\u00a0J. Syed, W.\u00a0T. Hole, and L.\u00a0C. Kingsland. 1993. The ranking algorithm of the Coach browser for the UMLS metathesaurus.Proceedings \/ the ... Annual Symposium on Computer Application [sic] in Medical Care. Symposium on Computer Applications in Medical Care (1993)."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1807184115"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1177\/0018720815625744"},{"key":"e_1_3_2_2_20_1","volume-title":"Grammarly: Free Writing Assistant.","author":"Inc. Grammarly.","year":"2018","unstructured":"Inc. Grammarly. 2018. Grammarly: Free Writing Assistant."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1038\/sdata.2016.35"},{"key":"e_1_3_2_2_22_1","unstructured":"Tom Kenter and Diana Maynard. 2004. Using Gate as an Annotation Tool. 1\u201311. https:\/\/gate.ac.uk\/sale\/am\/annotationmanual-gate2.pdf"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","unstructured":"Jon Kleinberg Himabindu Lakkaraju Jure Leskovec Jens Ludwig and Sendhil Mullainathan. 2018. Human decisions and machine predictions. Quarterly Journal of Economics(2018). https:\/\/doi.org\/10.1093\/qje\/qjx032","DOI":"10.1093\/qje\/qjx032"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","unstructured":"Chinmay Kulkarni Steven\u00a0P. Dow and Scott\u00a0R. Klemmer. 2014. Early and repeated exposure to examples improves creative work. In Design Thinking Research: Building Innovation Eco-Systems. https:\/\/doi.org\/10.1007\/978-3-319-01303-9_4","DOI":"10.1007\/978-3-319-01303-9_4"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","unstructured":"John\u00a0D. Lee and Katrina\u00a0A. See. 2004. Trust in automation: Designing for appropriate reliance. https:\/\/doi.org\/10.1518\/hfes.46.1.50_30392","DOI":"10.1518\/hfes.46.1.50_30392"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","unstructured":"Elisa\u00a0G. Liberati Francesca Ruggiero Laura Galuppo Mara Gorli Marien Gonz\u00e1lez-Lorenzo Marco Maraldi Pietro Ruggieri Hernan\u00a0Polo Friz Giuseppe Scaratti Koren\u00a0H. Kwag Roberto Vespignani and Lorenzo Moja. 2017. What hinders the uptake of computerized decision support systems in hospitals? A qualitative study and framework for implementation. Implementation Science(2017). https:\/\/doi.org\/10.1186\/s13012-017-0644-2","DOI":"10.1186\/s13012-017-0644-2"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","unstructured":"Todd Lingren Louise Deleger Katalin Molnar Haijun Zhai Jareen Meinzen-Derr Megan Kaiser Laura Stoutenborough Qi Li and Imre Solti. 2014. Evaluating the impact of pre-annotation on annotation speed and potential bias: Natural language processing gold standard development for clinical named entity recognition in clinical trial announcements. Journal of the American Medical Informatics Association (2014). https:\/\/doi.org\/10.1136\/amiajnl-2013-001837","DOI":"10.1136\/amiajnl-2013-001837"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbi.2019.103132"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073427.1073436"},{"key":"e_1_3_2_2_30_1","unstructured":"Nagarajan Natarajan Inderjit\u00a0S. Dhillon Pradeep Ravikumar and Ambuj Tewari. 2013. Learning with noisy labels. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2011-000116"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","unstructured":"Mark Neumann Daniel King Iz Beltagy and Waleed Ammar. 2019. ScispaCy: Fast and Robust Models for Biomedical Natural Language Processing. https:\/\/doi.org\/10.18653\/v1\/w19-5034 arxiv:1902.07669","DOI":"10.18653\/v1\/w19-5034"},{"key":"e_1_3_2_2_33_1","unstructured":"NIH-NLM. 2015. SNOMED Clinical Terms\u00ae (SNOMED CT\u00ae)."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"crossref","unstructured":"Mahsan Nourani Joanie\u00a0T. King and Eric\u00a0D. Ragan. 2020. The role of domain expertise in user trust and the impact of first impressions with intelligent systems. arxiv:2008.09100","DOI":"10.1609\/hcomp.v8i1.7469"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.3115\/1225785.1225791"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300293"},{"key":"e_1_3_2_2_37_1","unstructured":"Andrea Papenmeier Gwenn Englebienne and Christin Seifert. 2019. How model accuracy and explanation fidelity influence user trust in AI. arxiv:1907.12652"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1228"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","unstructured":"Andrey\u00a0A. Povyakalo Eugenio Alberdi Lorenzo Strigini and Peter Ayton. 2013. How to discriminate between computer-aided and computer-hindered decisions: A case study in mammography. Medical Decision Making(2013). https:\/\/doi.org\/10.1177\/0272989X12465490","DOI":"10.1177\/0272989X12465490"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/s14-2007"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.14778\/3157794.3157797"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1136\/jamia.2009.001560"},{"key":"e_1_3_2_2_43_1","unstructured":"D. Sculley Gary Holt Daniel Golovin Eugene Davydov Todd Phillips Dietmar Ebner Vinay Chaudhary Michael Young Jean\u00a0Fran\u00e7ois Crespo and Dan Dennison. 2015. Hidden technical debt in machine learning systems. In Advances in Neural Information Processing Systems."},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/2757226.2757230"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"crossref","unstructured":"Brett\u00a0R South Danielle Mowery Ying Suo Jianwei Leng Oscar Ferrandez Stephane\u00a0M Meystre and Wendy Champan. 2014. Evaluating the effects of machine pre-annotation and an interactive annotation interface on manual de-identification of clinical text.","DOI":"10.1016\/j.jbi.2014.05.002"},{"key":"e_1_3_2_2_46_1","volume-title":"EACL 2012 - Proceedings of the Demonstrations at the 13th Conference of the European Chapter of the Association for Computational Linguistics.","author":"Stenetorp Pontus","year":"2012","unstructured":"Pontus Stenetorp, Sampo Pyysalo, Goran Top\u00edc, Tomoko Ohta, Sophia Ananiadou, and Jun\u2019ichi Tsujii. 2012. BRAT: AWeb-based tool for NLP-Assisted text annotation. In EACL 2012 - Proceedings of the Demonstrations at the 13th Conference of the European Chapter of the Association for Computational Linguistics."},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-40802-1_24"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394231.3397922"},{"key":"e_1_3_2_2_49_1","unstructured":"Dayong Wang Aditya Khosla Rishab Gargeya Humayun Irshad and Andrew\u00a0H Beck. 2016. Deep Learning for Identifying Metastatic Breast Cancer. arXiv preprint arXiv:1606.05718(2016)."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1978963"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-4006"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/P14-5016"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300509"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","unstructured":"Yunfeng Zhang Q. Vera Liao and Rachel\u00a0K.E. Bellamy. 2020. Efect of confidence and explanation on accuracy and trust calibration in AI-assisted decision making. In FAT* 2020 - Proceedings of the 2020 Conference on Fairness Accountability and Transparency. https:\/\/doi.org\/10.1145\/3351095.3372852 arxiv:2001.02114","DOI":"10.1145\/3351095.3372852"}],"event":{"name":"CHI '21: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '21","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3411764.3445522","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3411764.3445522","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,21]],"date-time":"2026-09-21T13:54:20Z","timestamp":1789998860000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3411764.3445522"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,6]]},"references-count":54,"alternative-id":["10.1145\/3411764.3445522","10.1145\/3411764"],"URL":"https:\/\/doi.org\/10.1145\/3411764.3445522","relation":{},"subject":[],"published":{"date-parts":[[2021,5,6]]},"assertion":[{"value":"2021-05-07","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}