{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T15:22:23Z","timestamp":1776093743729,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":65,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,4,21]],"date-time":"2020-04-21T00:00:00Z","timestamp":1587427200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"NSERC CHRP","award":["CHRP 478468-15"],"award-info":[{"award-number":["CHRP 478468-15"]}]},{"name":"Google PhD Fellowship Program"},{"name":"CIHR CHRP","award":["CPG-140200"],"award-info":[{"award-number":["CPG-140200"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,4,21]]},"DOI":"10.1145\/3313831.3376506","type":"proceedings-article","created":{"date-parts":[[2020,5,27]],"date-time":"2020-05-27T16:16:47Z","timestamp":1590596207000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":35,"title":["Ambiguity-aware AI Assistants for Medical Data Analysis"],"prefix":"10.1145","author":[{"given":"Mike","family":"Schaekermann","sequence":"first","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]},{"given":"Graeme","family":"Beaton","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]},{"given":"Elaheh","family":"Sanoubari","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]},{"given":"Andrew","family":"Lim","sequence":"additional","affiliation":[{"name":"Sunnybrook Health Sciences Centre, Toronto, ON, Canada"}]},{"given":"Kate","family":"Larson","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]},{"given":"Edith","family":"Law","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]}],"member":"320","published-online":{"date-parts":[[2020,4,23]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174156"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.15346\/hc.v1i1.3"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.clinph.2017.06.252"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","unstructured":"A. Baker K. Young J. Potter and I. Madan. 2010. A review of grading systems for evidence-based guidelines produced by medical specialties. Clinical Medicine 10 4 (8 2010) 358--363. DOI: http:\/\/dx.doi.org\/10.7861\/clinmedicine.10--4--358","DOI":"10.7861\/clinmedicine.10--4--358"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1001\/jamanetworkopen.2019.0096"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1046\/j.1365--2753.2001.00284.x"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467--6494.1962.tb02303.x"},{"key":"e_1_3_2_1_8_1","unstructured":"Carrie J. Cai Emily Reif Narayan Hegde Jason Hipp Been Kim Daniel Smilkov Martin Wattenberg Fernanda Viegas Greg S. Corrado Martin C. Stumpe and Michael Terry. 2019a. Human-Centered Tools for Coping with Imperfect Algorithms during Medical Decision-Making. http:\/\/arxiv.org\/abs\/1902.02960"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359206"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.5555\/2540128.2540491"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300761"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.5555\/3306127.3331890"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.2018.4385"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0000000000001559"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP).","author":"Drapeau Ryan","unstructured":"Ryan Drapeau, Lydia B. Chilton, Jonathan Bragg, and Daniel S. Weld. 2016. MicroTalk: Using Argumentation to Improve Crowdsourcing Accuracy. In Proceedings of the 4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artmed.2004.07.007"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302316"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISBI.2019.8759380"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2016.2545658"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","unstructured":"Cosima Gretton. 2018. Trust and Transparency in Machine Learning-Based Clinical Decision Support. 279--292. DOI: http:\/\/dx.doi.org\/10.1007\/978--3--319--90403-0{_}14","DOI":"10.1007\/978--3--319--90403-0"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11756"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025781"},{"key":"e_1_3_2_1_24_1","first-page":"164","article-title":"A Study of Feature-based Consensus Formation for Glaucoma Risk Assessment","volume":"60","author":"Hammel Naama","year":"2019","unstructured":"Naama Hammel, Mike Schaekermann, Sonia Phene, Carter Dunn, Lily Peng, Dale R Webster, and Rory Sayres. 2019. A Study of Feature-based Consensus Formation for Glaucoma Risk Assessment. Investigative Ophthalmology & Visual Science 60, 9 (2019), 164.","journal-title":"Investigative Ophthalmology & Visual Science"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0166--4115(08)62386--9"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jrp.2019.02.003"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15327566IJCE0401"},{"key":"e_1_3_2_1_28_1","first-page":"313","article-title":"Majority Vote of Ensemble Machine Learning Methods for Real-Time Epilepsy Prediction Applied on EEG Pediatric Data","volume":"7","author":"Jasmin Samed Juki\u00b4","year":"2018","unstructured":"Samed Juki\u00b4 c and Jasmin Kevri\u00b4 c. 2018. Majority Vote of Ensemble Machine Learning Methods for Real-Time Epilepsy Prediction Applied on EEG Pediatric Data. TEM Journal 7, 2 (2018), 313.","journal-title":"TEM Journal"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858558"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0001--6918(99)00034--7"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300641"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ophtha.2018.01.034"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0b013e31821da915"},{"key":"e_1_3_2_1_34_1","volume-title":"WingIt: Efficient Refinement of Unclear Task Instructions. In The Sixth AAAI Conference on Human Computation and Crowdsourcing. 108--116","author":"Chaithanya Manam V K","year":"2018","unstructured":"V K Chaithanya Manam and Alexander J Quinn. 2018. WingIt: Efficient Refinement of Unclear Task Instructions. In The Sixth AAAI Conference on Human Computation and Crowdsourcing. 108--116. https:\/\/www.aaai.org\/ocs\/index.php\/HCOMP\/HCOMP18\/paper\/view\/17931"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287574"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1017\/9781316480748.019"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1197\/jamia.M2334"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2017.2647944"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ophtha.2019.07.024"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2018.00028"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173677"},{"key":"e_1_3_2_1_42_1","volume-title":"Direct Uncertainty Prediction for Medical Second Opinions. (7","author":"Raghu Maithra","year":"2018","unstructured":"Maithra Raghu, Katy Blumer, Rory Sayres, Ziad Obermeyer, Robert Kleinberg, Sendhil Mullainathan, and Jon Kleinberg. 2018. Direct Uncertainty Prediction for Medical Second Opinions. (7 2018). http:\/\/arxiv.org\/abs\/1807.01771"},{"key":"e_1_3_2_1_43_1","volume-title":"Ng","author":"Rajpurkar Pranav","year":"2017","unstructured":"Pranav Rajpurkar, Awni Y. Hannun, Masoumeh Haghpanahi, Codie Bourn, and Andrew Y. Ng. 2017. Cardiologist-Level Arrhythmia Detection with Convolutional Neural Networks. (7 2017). http:\/\/arxiv.org\/abs\/1707.01836"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-019-0099--8"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"D. A. Redelmeier and E. Shafir. 1995. Medical decision making in situations that offer multiple alternatives. JAMA 273 4 (1 1995) 302--5. http:\/\/www.ncbi.nlm.nih.gov\/pubmed\/7815657","DOI":"10.1001\/jama.1995.03520280048038"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.5664\/jcsm.2350"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/HRI.2019.8673326"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317085"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359178"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376290"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274423"},{"key":"e_1_3_2_1_52_1","first-page":"158","article-title":"Asynchronous Remote Adjudication for Grading Diabetic Retinopathy","volume":"60","author":"Schaekermann Mike","year":"2019","unstructured":"Mike Schaekermann, Naama Hammel, Brian Basham, Bilson Campana, Edith Law, Lily Peng, Dale R Webster, and Rory Sayres. 2019a. Asynchronous Remote Adjudication for Grading Diabetic Retinopathy. Investigative Ophthalmology & Visual Science 60, 9 (2019), 158.","journal-title":"Investigative Ophthalmology & Visual Science"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1167\/tvst.8.6.40"},{"key":"e_1_3_2_1_54_1","volume-title":"Expert Disagreement in Sequential Labeling: A Case Study on Adjudication in Medical Time Series Analysis. In 1st Workshop on Subjectivity, Ambiguity and Disagreement in Crowdsourcing at HCOMP","author":"Schaekermann Mike","year":"2018","unstructured":"Mike Schaekermann, Edith Law, Kate Larson, and Andrew Lim. 2018. Expert Disagreement in Sequential Labeling: A Case Study on Adjudication in Medical Time Series Analysis. In 1st Workshop on Subjectivity, Ambiguity and Disagreement in Crowdsourcing at HCOMP 2018. Zurich, Switzerland."},{"key":"e_1_3_2_1_55_1","volume-title":"Irresolvable Ambiguity: A New Hybrid Framework for Dealing with Uncertain Ground Truth. In 1st Workshop on Human-Centered Machine Learning at SIGCHI","author":"Schaekermann Mike","year":"2016","unstructured":"Mike Schaekermann, Edith Law, Alex C Williams, and William Callaghan. 2016. Resolvable vs. Irresolvable Ambiguity: A New Hybrid Framework for Dealing with Uncertain Ground Truth. In 1st Workshop on Human-Centered Machine Learning at SIGCHI 2016. San Jose, CA."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300391"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","unstructured":"Jens B. Stephansen Alexander N. Olesen Mads Olsen Aditya Ambati Eileen B. Leary Hyatt E. Moore Oscar Carrillo Ling Lin Fang Han Han Yan Yun L. Sun Yves Dauvilliers Sabine Scholz Lucie Barateau Birgit Hogl Ambra Stefani Seung Chul Hong Tae Won Kim Fabio Pizza Giuseppe Plazzi Stefano Vandi Elena Antelmi Dimitri Perrin Samuel T. Kuna Paula K. Schweitzer Clete Kushida Paul E. Peppard Helge B. D. Sorensen Poul Jennum and Emmanuel Mignot. 2018. Neural network analysis of sleep stages enables efficient diagnosis of narcolepsy. Nature Communications 9 1 (12 2018) 5229. DOI: http:\/\/dx.doi.org\/10.1038\/s41467-018-07229--3","DOI":"10.1038\/s41467-018-07229--3"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1136\/bmjopen-2013-002700"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.185.4157.1124"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1098\/rsos.181870"},{"key":"e_1_3_2_1_61_1","volume-title":"Proceedings of the national conference on artificial intelligence. Menlo Park, CA; Cambridge, MA; London; AAAI Press; MIT Press;","author":"Lent Michael Van","year":"2004","unstructured":"Michael Van Lent, William Fisher, and Michael Mancuso. 2004. An explainable artificial intelligence system for small-unit tactical behavior. In Proceedings of the national conference on artificial intelligence. Menlo Park, CA; Cambridge, MA; London; AAAI Press; MIT Press; 1999, 900--907."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300831"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1038\/nmeth.2855"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"crossref","unstructured":"Jennifer Williams Fod\u00e9 Abass Cisse Mike Schaekermann Foksuna Sakadi Nana Rahamatou Tassiou Aissatou Kenda BAH Abdoul Bachir Djibo Hamani Andrew Lim Edward C W Leung Tadeu A Fantaneau Tracey Milligan Vidita Khatri Daniel Hoch Manav Vyas Alice Lam Gladia Hotan Joseph Cohen Edith Law and Farrah Mateen. 2019a. Utilizing a wearable smartphone-based EEG for pediatric epilepsy patients in the resource poor environment of Guinea: A prospective study. Neurology 92 15 Supplement (2019). https:\/\/n.neurology.org\/content\/92\/15_Supplement\/N5.001","DOI":"10.1212\/WNL.92.15_supplement.N5.001"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","unstructured":"Jennifer A Williams Fod\u00e9 Abass Cisse Mike Schaekermann Foksouna Sakadi Nana Rahamatou Tassiou Gladia C. Hotan Aissatou Kenda Bah Abdoul Bachir Djibo Hamani Andrew Lim Edward C.W. Leung Tadeu A. Fantaneanu Tracey A. Milligan Vidita Khatri Daniel B. Hoch Manav V. Vyas Alice D. Lam Joseph M. Cohen Andre C. Vogel Edith Law and Farrah J. Mateen. 2019b. Smartphone EEG and remote online interpretation for children with epilepsy in the Republic of Guinea: Quality characteristics and practice implications. Seizure 71 (10 2019) 93--99. DOI: http:\/\/dx.doi.org\/10.1016\/j.seizure.2019.05.025","DOI":"10.1016\/j.seizure.2019.05.025"}],"event":{"name":"CHI '20: CHI Conference on Human Factors in Computing Systems","location":"Honolulu HI USA","acronym":"CHI '20","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3313831.3376506","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3313831.3376506","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:32:52Z","timestamp":1750199572000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3313831.3376506"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,4,21]]},"references-count":65,"alternative-id":["10.1145\/3313831.3376506","10.1145\/3313831"],"URL":"https:\/\/doi.org\/10.1145\/3313831.3376506","relation":{},"subject":[],"published":{"date-parts":[[2020,4,21]]},"assertion":[{"value":"2020-04-23","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}