{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T16:41:09Z","timestamp":1781887269927,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":64,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,19]],"date-time":"2023-04-19T00:00:00Z","timestamp":1681862400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100020959","name":"JST-Mirai Program Grant","doi-asserted-by":"publisher","award":["No.JPMJMI20B8"],"award-info":[{"award-number":["No.JPMJMI20B8"]}],"id":[{"id":"10.13039\/501100020959","id-type":"DOI","asserted-by":"publisher"}]},{"name":"NSERC","award":["No.RGPIN-2021-02549, No.RGPAS-2021-00034, No.DGECR-2021-00019"],"award-info":[{"award-number":["No.RGPIN-2021-02549, No.RGPAS-2021-00034, No.DGECR-2021-00019"]}]},{"name":"JSPS KAKENHI Grant","award":["No.JP20H04168"],"award-info":[{"award-number":["No.JP20H04168"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,19]]},"DOI":"10.1145\/3544548.3580741","type":"proceedings-article","created":{"date-parts":[[2023,4,20]],"date-time":"2023-04-20T04:27:20Z","timestamp":1681964840000},"page":"1-17","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["DeepLens: Interactive Out-of-distribution Data Detection in NLP Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9267-4229","authenticated-orcid":false,"given":"Da","family":"Song","sequence":"first","affiliation":[{"name":"University of Alberta, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4559-5426","authenticated-orcid":false,"given":"Zhijie","family":"Wang","sequence":"additional","affiliation":[{"name":"University of Alberta, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3666-4020","authenticated-orcid":false,"given":"Yuheng","family":"Huang","sequence":"additional","affiliation":[{"name":"University of Alberta, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8621-2420","authenticated-orcid":false,"given":"Lei","family":"Ma","sequence":"additional","affiliation":[{"name":"University of Alberta, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5468-9347","authenticated-orcid":false,"given":"Tianyi","family":"Zhang","sequence":"additional","affiliation":[{"name":"Computer Science, Purdue University, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,4,19]]},"reference":[{"key":"e_1_3_3_3_1_1","doi-asserted-by":"crossref","unstructured":"J Alammar. 2021. Ecco: An Open Source Library for the Explainability of Transformer Language Models. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing: System Demonstrations. Association for Computational Linguistics.","DOI":"10.18653\/v1\/2021.acl-demo.30"},{"key":"e_1_3_3_3_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"e_1_3_3_3_3_1","unstructured":"Dario Amodei Chris Olah Jacob Steinhardt Paul Christiano John Schulman and Dan Man\u00e9. 2016. Concrete problems in AI safety. arXiv preprint arXiv:1606.06565(2016)."},{"key":"e_1_3_3_3_4_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.835"},{"key":"e_1_3_3_3_5_1","first-page":"1179","article-title":"Single Layer Predictive Normalized Maximum Likelihood for Out-of-Distribution Detection","volume":"34","author":"Bibas Koby","year":"2021","unstructured":"Koby Bibas, Meir Feder, and Tal Hassner. 2021. Single Layer Predictive Normalized Maximum Likelihood for Out-of-Distribution Detection. Advances in Neural Information Processing Systems 34 (2021), 1179\u20131191.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_3_6_1","volume-title":"Natural language processing with Python: analyzing text with the natural language toolkit. \" O\u2019Reilly Media","author":"Bird Steven","unstructured":"Steven Bird, Ewan Klein, and Edward Loper. 2009. Natural language processing with Python: analyzing text with the natural language toolkit. \" O\u2019Reilly Media, Inc.\"."},{"key":"e_1_3_3_3_7_1","first-page":"16085","article-title":"Certifiably adversarially robust detection of out-of-distribution data","volume":"33","author":"Bitterwolf Julian","year":"2020","unstructured":"Julian Bitterwolf, Alexander Meinke, and Matthias Hein. 2020. Certifiably adversarially robust detection of out-of-distribution data. Advances in Neural Information Processing Systems 33 (2020), 16085\u201316095.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_3_8_1","volume-title":"Rasa: Open source language understanding and dialogue management. arXiv preprint arXiv:1712.05181(2017).","author":"Bocklisch Tom","year":"2017","unstructured":"Tom Bocklisch, Joey Faulkner, Nick Pawlowski, and Alan Nichol. 2017. Rasa: Open source language understanding and dialogue management. arXiv preprint arXiv:1712.05181(2017)."},{"key":"e_1_3_3_3_9_1","volume-title":"Oodanalyzer: Interactive analysis of out-of-distribution samples","author":"Chen Changjian","year":"2020","unstructured":"Changjian Chen, Jun Yuan, Yafeng Lu, Yang Liu, Hang Su, Songtao Yuan, and Shixia Liu. 2020. Oodanalyzer: Interactive analysis of out-of-distribution samples. IEEE transactions on visualization and computer graphics 27, 7(2020), 3335\u20133349."},{"key":"e_1_3_3_3_10_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR)","author":"Choi Sungik","year":"2020","unstructured":"Sungik Choi and Sae-Young Chung. 2020. Novelty detection via blurring. Proceedings of the International Conference on Learning Representations (ICLR) (2020)."},{"key":"e_1_3_3_3_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-73696-5_16"},{"key":"e_1_3_3_3_12_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018).","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805(2018)."},{"key":"e_1_3_3_3_13_1","volume-title":"Using icon arrays to communicate medical risks: overcoming low numeracy.Health Psychology 28, 2","author":"Galesic Mirta","year":"2009","unstructured":"Mirta Galesic, Rocio Garcia-Retamero, and Gerd Gigerenzer. 2009. Using icon arrays to communicate medical risks: overcoming low numeracy.Health Psychology 28, 2 (2009), 210."},{"key":"e_1_3_3_3_14_1","volume-title":"Explaining and Harnessing Adversarial Examples. In International Conference on Learning Representations. http:\/\/arxiv.org\/abs\/1412","author":"Goodfellow Ian","year":"2015","unstructured":"Ian Goodfellow, Jonathon Shlens, and Christian Szegedy. 2015. Explaining and Harnessing Adversarial Examples. In International Conference on Learning Representations. http:\/\/arxiv.org\/abs\/1412.6572"},{"key":"e_1_3_3_3_15_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.95.5.2703"},{"key":"e_1_3_3_3_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2089094.2089099"},{"key":"e_1_3_3_3_17_1","volume-title":"International conference on machine learning. PMLR, 1321\u20131330","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q Weinberger. 2017. On calibration of modern neural networks. In International conference on machine learning. PMLR, 1321\u20131330."},{"key":"e_1_3_3_3_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2017.2743959"},{"key":"e_1_3_3_3_19_1","volume-title":"Advances in psychology. Vol.\u00a052","author":"Hart G","unstructured":"Sandra\u00a0G Hart and Lowell\u00a0E Staveland. 1988. Development of NASA-TLX (Task Load Index): Results of empirical and theoretical research. In Advances in psychology. Vol.\u00a052. Elsevier, 139\u2013183."},{"key":"e_1_3_3_3_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00013"},{"key":"e_1_3_3_3_21_1","volume-title":"5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings. OpenReview.net. https:\/\/openreview.net\/forum?id=Hkg4TI9xl","author":"Hendrycks Dan","year":"2017","unstructured":"Dan Hendrycks and Kevin Gimpel. 2017. A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks. In 5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings. OpenReview.net. https:\/\/openreview.net\/forum?id=Hkg4TI9xl"},{"key":"e_1_3_3_3_22_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR)","author":"Hendrycks Dan","year":"2020","unstructured":"Dan Hendrycks, Norman Mu, Ekin\u00a0D. Cubuk, Barret Zoph, Justin Gilmer, and Balaji Lakshminarayanan. 2020. AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty. Proceedings of the International Conference on Learning Representations (ICLR) (2020)."},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1275"},{"key":"e_1_3_3_3_24_1","volume-title":"Stochastic neighbor embedding. Advances in neural information processing systems 15","author":"Hinton E","year":"2002","unstructured":"Geoffrey\u00a0E Hinton and Sam Roweis. 2002. Stochastic neighbor embedding. Advances in neural information processing systems 15 (2002)."},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"crossref","unstructured":"Benjamin Hoover Hendrik Strobelt and Sebastian Gehrmann. 2019. exbert: A visual analysis tool to explore learned representations in transformers models. arXiv preprint arXiv:1910.05276(2019).","DOI":"10.18653\/v1\/2020.acl-demos.22"},{"key":"e_1_3_3_3_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.01096"},{"key":"e_1_3_3_3_27_1","volume-title":"Principal component analysis for special types of data","author":"Jolliffe T","unstructured":"Ian\u00a0T Jolliffe. 2002. Principal component analysis for special types of data. Springer."},{"key":"e_1_3_3_3_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858558"},{"key":"e_1_3_3_3_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_3_3_30_1","volume-title":"The parable of Google Flu: traps in big data analysis. Science 343, 6176","author":"Lazer David","year":"2014","unstructured":"David Lazer, Ryan Kennedy, Gary King, and Alessandro Vespignani. 2014. The parable of Google Flu: traps in big data analysis. Science 343, 6176 (2014), 1203\u20131205."},{"key":"e_1_3_3_3_31_1","volume-title":"A simple unified framework for detecting out-of-distribution samples and adversarial attacks. Advances in neural information processing systems 31","author":"Lee Kimin","year":"2018","unstructured":"Kimin Lee, Kibok Lee, Honglak Lee, and Jinwoo Shin. 2018. A simple unified framework for detecting out-of-distribution samples and adversarial attacks. Advances in neural information processing systems 31 (2018)."},{"key":"e_1_3_3_3_32_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10443-0_39"},{"key":"e_1_3_3_3_33_1","volume-title":"Enhancing The Reliability of Out-of-distribution Image Detection in Neural Networks. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=H1VGkIxRZ","author":"Liang Shiyu","unstructured":"Shiyu Liang, Yixuan Li, and R. Srikant. 2018. Enhancing The Reliability of Out-of-distribution Image Detection in Neural Networks. In International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=H1VGkIxRZ"},{"key":"e_1_3_3_3_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01506"},{"key":"e_1_3_3_3_35_1","first-page":"21464","article-title":"Energy-based out-of-distribution detection","volume":"33","author":"Liu Weitang","year":"2020","unstructured":"Weitang Liu, Xiaoyun Wang, John Owens, and Yixuan Li. 2020. Energy-based out-of-distribution detection. Advances in Neural Information Processing Systems 33 (2020), 21464\u201321475.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_3_3_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1982.1056489"},{"key":"e_1_3_3_3_37_1","volume-title":"International Conference on Machine Learning. PMLR, 2218\u20132227","author":"Louizos Christos","year":"2017","unstructured":"Christos Louizos and Max Welling. 2017. Multiplicative normalizing flows for variational bayesian neural networks. In International Conference on Machine Learning. PMLR, 2218\u20132227."},{"key":"e_1_3_3_3_38_1","first-page":"2346","article-title":"Learning under concept drift: A review","volume":"31","author":"Lu Jie","year":"2018","unstructured":"Jie Lu, Anjin Liu, Fan Dong, Feng Gu, Joao Gama, and Guangquan Zhang. 2018. Learning under concept drift: A review. IEEE Transactions on Knowledge and Data Engineering 31, 12(2018), 2346\u20132363.","journal-title":"IEEE Transactions on Knowledge and Data Engineering"},{"key":"e_1_3_3_3_39_1","doi-asserted-by":"publisher","DOI":"10.5555\/2002472.2002491"},{"key":"e_1_3_3_3_40_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR)","author":"Meinke Alexander","year":"2020","unstructured":"Alexander Meinke and Matthias Hein. 2020. Towards neural networks that provably know when they don\u2019t know. Proceedings of the International Conference on Learning Representations (ICLR) (2020)."},{"key":"e_1_3_3_3_41_1","volume-title":"Explanation in artificial intelligence: Insights from the social sciences. Artificial intelligence 267","author":"Miller Tim","year":"2019","unstructured":"Tim Miller. 2019. Explanation in artificial intelligence: Insights from the social sciences. Artificial intelligence 267 (2019), 1\u201338."},{"key":"e_1_3_3_3_42_1","volume-title":"Foundations of machine learning","author":"Mohri Mehryar","unstructured":"Mehryar Mohri, Afshin Rostamizadeh, and Ameet Talwalkar. 2018. Foundations of machine learning. MIT press."},{"key":"e_1_3_3_3_43_1","volume-title":"A unifying view on dataset shift in classification. Pattern recognition 45, 1","author":"Moreno-Torres G","year":"2012","unstructured":"Jose\u00a0G Moreno-Torres, Troy Raeder, Roc\u00edo Alaiz-Rodr\u00edguez, Nitesh\u00a0V Chawla, and Francisco Herrera. 2012. A unifying view on dataset shift in classification. Pattern recognition 45, 1 (2012), 521\u2013530."},{"key":"e_1_3_3_3_44_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i7.20752"},{"key":"e_1_3_3_3_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"e_1_3_3_3_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/VIS49827.2021.9623289"},{"key":"e_1_3_3_3_47_1","unstructured":"Jo\u00e3o Palmeiro Beatriz Malveiro Rita Costa David Polido Ricardo Moreira and Pedro Bizarro. 2022. Data+ Shift: Supporting visual investigation of data distribution shifts by data scientists. arXiv preprint arXiv:2204.14025(2022)."},{"key":"e_1_3_3_3_48_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_3_3_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417063"},{"key":"e_1_3_3_3_50_1","volume-title":"The promises and pitfalls of machine learning for detecting viruses in aquatic metagenomes. Frontiers in microbiology 10","author":"Ponsero J","year":"2019","unstructured":"Alise\u00a0J Ponsero and Bonnie\u00a0L Hurwitz. 2019. The promises and pitfalls of machine learning for detecting viruses in aquatic metagenomes. Frontiers in microbiology 10 (2019), 806."},{"key":"e_1_3_3_3_51_1","volume-title":"Failing loudly: An empirical study of methods for detecting dataset shift. Advances in Neural Information Processing Systems 32","author":"Rabanser Stephan","year":"2019","unstructured":"Stephan Rabanser, Stephan G\u00fcnnemann, and Zachary Lipton. 2019. Failing loudly: An empirical study of methods for detecting dataset shift. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_3_3_52_1","volume-title":"Likelihood ratios for out-of-distribution detection. Advances in neural information processing systems 32","author":"Ren Jie","year":"2019","unstructured":"Jie Ren, Peter\u00a0J Liu, Emily Fertig, Jasper Snoek, Ryan Poplin, Mark Depristo, Joshua Dillon, and Balaji Lakshminarayanan. 2019. Likelihood ratios for out-of-distribution detection. Advances in neural information processing systems 32 (2019)."},{"key":"e_1_3_3_3_53_1","doi-asserted-by":"publisher","DOI":"10.1016\/0377-0427(87)90125-7"},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1404"},{"key":"e_1_3_3_3_55_1","volume-title":"Fakenewsnet: A data repository with news content, social context, and spatiotemporal information for studying fake news on social media. Big data 8, 3","author":"Shu Kai","year":"2020","unstructured":"Kai Shu, Deepak Mahudeswaran, Suhang Wang, Dongwon Lee, and Huan Liu. 2020. Fakenewsnet: A data repository with news content, social context, and spatiotemporal information for studying fake news on social media. Big data 8, 3 (2020), 171\u2013188."},{"key":"e_1_3_3_3_56_1","volume-title":"A global geometric framework for nonlinear dimensionality reduction. science 290, 5500","author":"Tenenbaum B","year":"2000","unstructured":"Joshua\u00a0B Tenenbaum, Vin\u00a0de Silva, and John\u00a0C Langford. 2000. A global geometric framework for nonlinear dimensionality reduction. science 290, 5500 (2000), 2319\u20132323."},{"key":"e_1_3_3_3_57_1","volume-title":"On mixup training: Improved calibration and predictive uncertainty for deep neural networks. Advances in Neural Information Processing Systems 32","author":"Thulasidasan Sunil","year":"2019","unstructured":"Sunil Thulasidasan, Gopinath Chennupati, Jeff\u00a0A Bilmes, Tanmoy Bhattacharya, and Sarah Michalak. 2019. On mixup training: Improved calibration and predictive uncertainty for deep neural networks. Advances in Neural Information Processing Systems 32 (2019)."},{"key":"e_1_3_3_3_58_1","volume-title":"Can multi-label classification networks know what they don\u2019t know?Advances in Neural Information Processing Systems 34","author":"Wang Haoran","year":"2021","unstructured":"Haoran Wang, Weitang Liu, Alex Bocchieri, and Yixuan Li. 2021. Can multi-label classification networks know what they don\u2019t know?Advances in Neural Information Processing Systems 34 (2021), 29074\u201329087."},{"key":"e_1_3_3_3_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST50239.2020.00006"},{"key":"e_1_3_3_3_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST50239.2020.00007"},{"key":"e_1_3_3_3_61_1","volume-title":"Visual drift detection for sequence data analysis of business processes","author":"Yeshchenko Anton","year":"2021","unstructured":"Anton Yeshchenko, Claudio Di\u00a0Ciccio, Jan Mendling, and Artem Polyvyanyy. 2021. Visual drift detection for sequence data analysis of business processes. IEEE Transactions on Visualization and Computer Graphics (2021)."},{"key":"e_1_3_3_3_62_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00612"},{"key":"e_1_3_3_3_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445396"},{"key":"e_1_3_3_3_64_1","volume-title":"Character-level convolutional networks for text classification. Advances in neural information processing systems 28","author":"Zhang Xiang","year":"2015","unstructured":"Xiang Zhang, Junbo Zhao, and Yann LeCun. 2015. Character-level convolutional networks for text classification. Advances in neural information processing systems 28 (2015)."}],"event":{"name":"CHI '23: CHI Conference on Human Factors in Computing Systems","location":"Hamburg Germany","acronym":"CHI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3580741","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3580741","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3580741","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:24Z","timestamp":1750178244000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3580741"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,19]]},"references-count":64,"alternative-id":["10.1145\/3544548.3580741","10.1145\/3544548"],"URL":"https:\/\/doi.org\/10.1145\/3544548.3580741","relation":{},"subject":[],"published":{"date-parts":[[2023,4,19]]},"assertion":[{"value":"2023-04-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}