{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,28]],"date-time":"2026-09-28T19:13:12Z","timestamp":1790622792799,"version":"4.1.0"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,3,22]],"date-time":"2022-03-22T00:00:00Z","timestamp":1647907200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,3,22]]},"DOI":"10.1145\/3490099.3511141","type":"proceedings-article","created":{"date-parts":[[2022,3,21]],"date-time":"2022-03-21T19:24:13Z","timestamp":1647890653000},"page":"549-565","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["HINT: Integration Testing for AI-based features with Humans in the Loop"],"prefix":"10.1145","author":[{"given":"Quan Ze","family":"Chen","sequence":"first","affiliation":[{"name":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tobias","family":"Schnabel","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Besmira","family":"Nushi","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Saleema","family":"Amershi","sequence":"additional","affiliation":[{"name":"Microsoft Research AI, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,3,22]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46454-1_24"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012429"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Gagan Bansal Tongshuang Wu Joyce Zhu Raymond Fok Besmira Nushi Ece Kamar Marco\u00a0Tulio Ribeiro and Daniel\u00a0S Weld. 2020. Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance. arXiv preprint arXiv:2006.14779(2020).  Gagan Bansal Tongshuang Wu Joyce Zhu Raymond Fok Besmira Nushi Ece Kamar Marco\u00a0Tulio Ribeiro and Daniel\u00a0S Weld. 2020. Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance. arXiv preprint arXiv:2006.14779(2020).","DOI":"10.1145\/3411764.3445717"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377498"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788613"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376835"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Jan Deriu Alvaro Rodrigo Arantxa Otegi Guillermo Echegoyen Sophie Rosset Eneko Agirre and Mark Cieliebak. 2020. Survey on evaluation methods for dialogue systems. Artificial Intelligence Review(2020) 1\u201356.  Jan Deriu Alvaro Rodrigo Arantxa Otegi Guillermo Echegoyen Sophie Rosset Eneko Agirre and Mark Cieliebak. 2020. Survey on evaluation methods for dialogue systems. Artificial Intelligence Review(2020) 1\u201356.","DOI":"10.1007\/s10462-020-09866-x"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2013.44"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2792838.2800195"},{"key":"e_1_3_2_1_13_1","unstructured":"Markus Freitag George\u00a0F. Foster David Grangier Viresh Ratnakar Qijun Tan and Wolfgang Macherey. 2021. Experts Errors and Context: A Large-Scale Study of Human Evaluation for Machine Translation. ArXiv abs\/2104.14478(2021).  Markus Freitag George\u00a0F. Foster David Grangier Viresh Ratnakar Qijun Tan and Wolfgang Macherey. 2021. Experts Errors and Context: A Large-Scale Study of Human Evaluation for Machine Translation. ArXiv abs\/2104.14478(2021)."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/289444.289502"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2645710.2645745"},{"key":"e_1_3_2_1_16_1","volume-title":"User Experience Evaluation through Automatic A\/B Testing(IUI \u201920)","author":"Gardey Juan\u00a0Cruz","unstructured":"Juan\u00a0Cruz Gardey and Alejandra Garrido . 2020. User Experience Evaluation through Automatic A\/B Testing(IUI \u201920) . Association for Computing Machinery , New York, NY, USA , 25\u201326. https:\/\/doi.org\/10.1145\/3379336.3381514 10.1145\/3379336.3381514 Juan\u00a0Cruz Gardey and Alejandra Garrido. 2020. User Experience Evaluation through Automatic A\/B Testing(IUI \u201920). Association for Computing Machinery, New York, NY, USA, 25\u201326. https:\/\/doi.org\/10.1145\/3379336.3381514"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Mor Geva Y. Goldberg and Jonathan Berant. 2019. Are We Modeling the Task or the Annotator? An Investigation of Annotator Bias in Natural Language Understanding Datasets. ArXiv abs\/1908.07898(2019).  Mor Geva Y. Goldberg and Jonathan Berant. 2019. Are We Modeling the Task or the Annotator? An Investigation of Annotator Bias in Natural Language Understanding Datasets. ArXiv abs\/1908.07898(2019).","DOI":"10.18653\/v1\/D19-1107"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1378773.1378804"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/169891.169892"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1177\/0018720814547570"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1229179.1229181"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858558"},{"key":"e_1_3_2_1_24_1","volume-title":"Human decisions and machine predictions. The quarterly journal of economics 133, 1","author":"Kleinberg Jon","year":"2018","unstructured":"Jon Kleinberg , Himabindu Lakkaraju , Jure Leskovec , Jens Ludwig , and Sendhil Mullainathan . 2018. Human decisions and machine predictions. The quarterly journal of economics 133, 1 ( 2018 ), 237\u2013293. Jon Kleinberg, Himabindu Lakkaraju, Jure Leskovec, Jens Ludwig, and Sendhil Mullainathan. 2018. Human decisions and machine predictions. The quarterly journal of economics 133, 1 (2018), 237\u2013293."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Bryan Klimt and Yiming Yang. 2004. The Enron Corpus: A New Dataset for Email Classification Research. In ECML. 217\u2013226.  Bryan Klimt and Yiming Yang. 2004. The Enron Corpus: A New Dataset for Email Classification Research. In ECML. 217\u2013226.","DOI":"10.1007\/978-3-540-30115-8_22"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300641"},{"key":"e_1_3_2_1_27_1","volume-title":"Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery 18, 1","author":"Kohavi Ron","year":"2009","unstructured":"Ron Kohavi , Roger Longbotham , Dan Sommerfield , and Randal\u00a0 M Henne . 2009. Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery 18, 1 ( 2009 ), 140\u2013181. Ron Kohavi, Roger Longbotham, Dan Sommerfield, and Randal\u00a0M Henne. 2009. Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery 18, 1 (2009), 140\u2013181."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470684"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3375627.3375833"},{"key":"e_1_3_2_1_30_1","volume-title":"Trust in automation: Designing for appropriate reliance. Human factors 46, 1","author":"Lee D","year":"2004","unstructured":"John\u00a0 D Lee and Katrina\u00a0 A See . 2004. Trust in automation: Designing for appropriate reliance. Human factors 46, 1 ( 2004 ), 50\u201380. John\u00a0D Lee and Katrina\u00a0A See. 2004. Trust in automation: Designing for appropriate reliance. Human factors 46, 1 (2004), 50\u201380."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1002\/meet.14504901134"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Reid McIlroy-Young Siddhartha Sen Jon Kleinberg and Ashton Anderson. 2020. Aligning Superhuman AI and Human Behavior: Chess as a Model System. arXiv preprint arXiv:2006.01855(2020).  Reid McIlroy-Young Siddhartha Sen Jon Kleinberg and Ashton Anderson. 2020. Aligning Superhuman AI and Human Behavior: Chess as a Model System. arXiv preprint arXiv:2006.01855(2020).","DOI":"10.1145\/3394486.3403219"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2494603.2480303"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/1102351.1102430"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"crossref","unstructured":"Jakob Nielsen. 1994. Usability engineering. Morgan Kaufmann.  Jakob Nielsen. 1994. Usability engineering. Morgan Kaufmann.","DOI":"10.1016\/B978-0-08-052029-2.50009-7"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v6i1.13337"},{"key":"e_1_3_2_1_37_1","first-page":"331","article-title":"Testing in software development","volume":"71","author":"Ould M.","year":"1987","unstructured":"M. Ould and Charles Unwin . 1987 . Testing in software development . The Mathematical Gazette 71 (1987), 331 \u2013 331 . M. Ould and Charles Unwin. 1987. Testing in software development. The Mathematical Gazette 71 (1987), 331\u2013331.","journal-title":"The Mathematical Gazette"},{"key":"e_1_3_2_1_38_1","unstructured":"Forough Poursabzi-Sangdeh Daniel\u00a0G Goldstein Jake\u00a0M Hofman Jennifer\u00a0Wortman Vaughan and Hanna Wallach. 2018. Manipulating and measuring model interpretability. arXiv preprint arXiv:1802.07810(2018).  Forough Poursabzi-Sangdeh Daniel\u00a0G Goldstein Jake\u00a0M Hofman Jennifer\u00a0Wortman Vaughan and Hanna Wallach. 2018. Manipulating and measuring model interpretability. arXiv preprint arXiv:1802.07810(2018)."},{"key":"e_1_3_2_1_39_1","unstructured":"Maithra Raghu Katy Blumer Greg Corrado Jon Kleinberg Ziad Obermeyer and Sendhil Mullainathan. 2019. The algorithmic automation problem: Prediction triage and human effort. arXiv preprint arXiv:1903.12220(2019).  Maithra Raghu Katy Blumer Greg Corrado Jon Kleinberg Ziad Obermeyer and Sendhil Mullainathan. 2019. The algorithmic automation problem: Prediction triage and human effort. arXiv preprint arXiv:1903.12220(2019)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"e_1_3_2_1_42_1","volume-title":"Finding and Eliciting Richer Preferences for Recommendation. In Fourteenth ACM Conference on Recommender Systems","author":"Schnabel Tobias","year":"2020","unstructured":"Tobias Schnabel , Gonzalo Ramos , and Saleema Amershi . 2020 . \u201c Who Doesn\u2019t like Dinosaurs? \u201d Finding and Eliciting Richer Preferences for Recommendation. In Fourteenth ACM Conference on Recommender Systems ( Virtual Event, Brazil) (RecSys \u201920). Association for Computing Machinery, New York, NY, USA, 398\u2013407. https:\/\/doi.org\/10.1145\/3383313.3412267 10.1145\/3383313.3412267 Tobias Schnabel, Gonzalo Ramos, and Saleema Amershi. 2020. \u201cWho Doesn\u2019t like Dinosaurs?\u201d Finding and Eliciting Richer Preferences for Recommendation. In Fourteenth ACM Conference on Recommender Systems(Virtual Event, Brazil) (RecSys \u201920). Association for Computing Machinery, New York, NY, USA, 398\u2013407. https:\/\/doi.org\/10.1145\/3383313.3412267"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/VL.1996.545307"},{"key":"e_1_3_2_1_44_1","volume-title":"Does automation bias decision-making?International Journal of Human-Computer Studies 51, 5","author":"Skitka J","year":"1999","unstructured":"Linda\u00a0 J Skitka , Kathleen\u00a0 L Mosier , and Mark Burdick . 1999. Does automation bias decision-making?International Journal of Human-Computer Studies 51, 5 ( 1999 ), 991\u20131006. Linda\u00a0J Skitka, Kathleen\u00a0L Mosier, and Mark Burdick. 1999. Does automation bias decision-making?International Journal of Human-Computer Studies 51, 5 (1999), 991\u20131006."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Dylan Slack Sophie Hilgard Emily Jia Sameer Singh and Himabindu Lakkaraju. 2019. How can we fool LIME and SHAP? Adversarial Attacks on Post hoc Explanation Methods. arXiv preprint arXiv:1911.02508(2019).  Dylan Slack Sophie Hilgard Emily Jia Sameer Singh and Himabindu Lakkaraju. 2019. How can we fool LIME and SHAP? Adversarial Attacks on Post hoc Explanation Methods. arXiv preprint arXiv:1911.02508(2019).","DOI":"10.1145\/3375627.3375830"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302292"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Andrew\u00a0H Turpin and William Hersh. 2001. Why batch and user evaluations do not give the same results. In SIGIR.  Andrew\u00a0H Turpin and William Hersh. 2001. Why batch and user evaluations do not give the same results. In SIGIR.","DOI":"10.1145\/383952.383992"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1177\/0272989X14547233"},{"key":"e_1_3_2_1_50_1","unstructured":"Chris Welty Praveen\u00a0K. Paritosh and Lora Aroyo. 2019. Metrology for AI: From Benchmarks to Instruments. ArXiv abs\/1911.01875(2019).  Chris Welty Praveen\u00a0K. Paritosh and Lora Aroyo. 2019. Metrology for AI: From Benchmarks to Instruments. ArXiv abs\/1911.01875(2019)."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/212"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3196709.3196730"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376301"},{"key":"e_1_3_2_1_54_1","volume-title":"Proceedings of the Sixth Workshop on Statistical Machine Translation. Association for Computational Linguistics","author":"Zaidan Omar","year":"2011","unstructured":"Omar Zaidan . 2011 . MAISE: A Flexible, Configurable, Extensible Open Source Package for Mass AI System Evaluation . In Proceedings of the Sixth Workshop on Statistical Machine Translation. Association for Computational Linguistics , Edinburgh, Scotland, 130\u2013134. https:\/\/aclanthology.org\/W11-2114 Omar Zaidan. 2011. MAISE: A Flexible, Configurable, Extensible Open Source Package for Mass AI System Evaluation. In Proceedings of the Sixth Workshop on Statistical Machine Translation. Association for Computational Linguistics, Edinburgh, Scotland, 130\u2013134. https:\/\/aclanthology.org\/W11-2114"},{"key":"e_1_3_2_1_55_1","volume-title":"Machine learning testing: Survey, landscapes and horizons","author":"Zhang M","year":"2020","unstructured":"Jie\u00a0 M Zhang , Mark Harman , Lei Ma , and Yang Liu . 2020. Machine learning testing: Survey, landscapes and horizons . IEEE Transactions on Software Engineering( 2020 ). Jie\u00a0M Zhang, Mark Harman, Lei Ma, and Yang Liu. 2020. Machine learning testing: Survey, landscapes and horizons. IEEE Transactions on Software Engineering(2020)."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372852"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11573-014-0721-9"}],"event":{"name":"IUI '22: 27th International Conference on Intelligent User Interfaces","location":"Helsinki Finland","acronym":"IUI '22","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["27th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3490099.3511141","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3490099.3511141","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T15:31:24Z","timestamp":1750174284000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3490099.3511141"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,3,22]]},"references-count":56,"alternative-id":["10.1145\/3490099.3511141","10.1145\/3490099"],"URL":"https:\/\/doi.org\/10.1145\/3490099.3511141","relation":{},"subject":[],"published":{"date-parts":[[2022,3,22]]},"assertion":[{"value":"2022-03-22","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}