{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,5]],"date-time":"2025-05-05T21:07:04Z","timestamp":1746479224420},"reference-count":34,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2020,11,1]],"date-time":"2020-11-01T00:00:00Z","timestamp":1604188800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,11,1]],"date-time":"2020-11-01T00:00:00Z","timestamp":1604188800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,11,1]],"date-time":"2020-11-01T00:00:00Z","timestamp":1604188800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Comput. Grap. Appl."],"published-print":{"date-parts":[[2020,11,1]]},"DOI":"10.1109\/mcg.2020.3017064","type":"journal-article","created":{"date-parts":[[2020,10,23]],"date-time":"2020-10-23T19:23:21Z","timestamp":1603481001000},"page":"88-96","source":"Crossref","is-referenced-by-count":12,"title":["Challenges in Evaluating Interactive Visual Machine Learning Systems"],"prefix":"10.1109","volume":"40","author":[{"given":"N.","family":"Boukhelifa","sequence":"first","affiliation":[{"name":"INRAE, Université Paris-Saclay"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"A.","family":"Bezerianos","sequence":"additional","affiliation":[{"name":"Université Paris-Saclay, CNRS, Inria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"R.","family":"Chang","sequence":"additional","affiliation":[{"name":"Tufts University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"C.","family":"Collins","sequence":"additional","affiliation":[{"name":"Ontario Tech University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"S.","family":"Drucker","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"A.","family":"Endert","sequence":"additional","affiliation":[{"name":"Georgia Tech"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"J.","family":"Hullman","sequence":"additional","affiliation":[{"name":"Northwestern University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"C.","family":"North","sequence":"additional","affiliation":[{"name":"Virginia Tech"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"M.","family":"Sedlmair","sequence":"additional","affiliation":[{"name":"University of Stuttgart"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2017.8585669"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2017.2744138"},{"key":"ref31","first-page":"641","article-title":"Human-centered machine learning through interactive visualization","author":"sacha","year":"2016","journal-title":"Proc 18th Eur Symp Artif Neural Netw Comput Intell Mach Learn"},{"key":"ref30","first-page":"19","article-title":"The Human user in progressive visual analytics","author":"micallef","year":"2019","journal-title":"Proc 21st EG\/VGTC Conf Visualization"},{"key":"ref34","first-page":"1","article-title":"On the cost of interactions in interactive visual machine learning","author":"zhang","year":"2019","journal-title":"Proc IEEE VIS Workshop Eval Interactive Visual Mach Learn Syst"},{"key":"ref10","author":"charmaz","year":"2006","journal-title":"Constructing grounded theory A practical guide through qualitative analysis"},{"key":"ref11","first-page":"1","article-title":"mVis in the wild: Pre-study of an interactive visual machine learning system for labelling","author":"chegini","year":"2019","journal-title":"Proc IEEE VIS workshop Workshop Eval Interactive Visual Mach Learn Syst"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1016\/j.visinf.2018.09.003"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2018.2872577"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-23765-2_13"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2017.2745080"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1007\/s10844-014-0304-9"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.13092"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173718"},{"key":"ref19","article-title":"Explainable artificial intelligence (XAI), Defense Advanced Research Projects Agency (DARPA)","author":"gunning","year":"2017"},{"key":"ref28","first-page":"59","article-title":"Human evaluation of models built for interpretability","volume":"7","author":"lage","year":"2019","journal-title":"Proc AAAI Conf Human Comput Crowdsourcing"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1162\/EVCO_a_00161"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/321906.321910"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/1562849.1562851"},{"key":"ref6","first-page":"1","article-title":"Evaluating semantic interaction on word embeddings via simulation","author":"bian","year":"2019","journal-title":"Proc IEEE Eval Interactive Visual Mach Learn Syst"},{"key":"ref29","article-title":"Users and customizable software: A co-adaptive phenomenon","author":"mackay","year":"1990"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-90403-0_17"},{"key":"ref8","first-page":"1","article-title":"Inferential tasks as a data-rich evaluation method for visualization","author":"cashman","year":"2019","journal-title":"Proc IEEE VIS Workshop Eval Interactive Visual Mach Learn Syst"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2012.6400486"},{"key":"ref2","first-page":"2","article-title":"Beyond accuracy: The role of mental models in human-AI team performance","volume":"7","author":"bansal","year":"2019","journal-title":"Proc AAAI Conf Human Comput Crowdsourcing"},{"key":"ref9","first-page":"189","article-title":"LIDeOGraM: An interactive evolutionary modelling tool","author":"chabin","year":"2017","journal-title":"Proc Int Conf Artif Evol (Evol Artificielle)"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/3392878"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2018.2864889"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/ARSO.2017.8025197"},{"key":"ref24","first-page":"1","article-title":"How does visualization help people learn deep learning","author":"kahng","year":"2019","journal-title":"Proc IEEE VIS workshop EVIVA-ML"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1007\/s12650-018-0531-1"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300912"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025592"}],"container-title":["IEEE Computer Graphics and Applications"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/38\/9238532\/09238590.pdf?arnumber=9238590","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,5,19]],"date-time":"2022-05-19T20:21:35Z","timestamp":1652991695000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9238590\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,11,1]]},"references-count":34,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/mcg.2020.3017064","relation":{},"ISSN":["0272-1716","1558-1756"],"issn-type":[{"value":"0272-1716","type":"print"},{"value":"1558-1756","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,11,1]]}}}