{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,26]],"date-time":"2025-12-26T07:05:53Z","timestamp":1766732753546,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,3,27]],"date-time":"2023-03-27T00:00:00Z","timestamp":1679875200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,3,27]]},"DOI":"10.1145\/3581641.3584096","type":"proceedings-article","created":{"date-parts":[[2023,3,27]],"date-time":"2023-03-27T16:16:52Z","timestamp":1679933812000},"page":"650-663","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["Perspective: Leveraging Human Understanding for Identifying and Characterizing Image Atypicality"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6211-1871","authenticated-orcid":false,"given":"Shahin","family":"Sharifi Noorian","sequence":"first","affiliation":[{"name":"Web Information Systems, Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1629-8473","authenticated-orcid":false,"given":"Sihang","family":"Qiu","sequence":"additional","affiliation":[{"name":"College of Systems Engineering, Hunan Institute of Advanced Technology, China"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6804-127X","authenticated-orcid":false,"given":"Burcu","family":"Sayin","sequence":"additional","affiliation":[{"name":"University of Trento, Italy"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2725-5305","authenticated-orcid":false,"given":"Agathe","family":"Balayn","sequence":"additional","affiliation":[{"name":"Web Information Systems, Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6189-6539","authenticated-orcid":false,"given":"Ujwal","family":"Gadiraju","sequence":"additional","affiliation":[{"name":"Web Information Systems, Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0350-0313","authenticated-orcid":false,"given":"Jie","family":"Yang","sequence":"additional","affiliation":[{"name":"Web Information Systems, Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3300-2913","authenticated-orcid":false,"given":"Alessandro","family":"Bozzon","sequence":"additional","affiliation":[{"name":"Web Information Systems, Delft University of Technology, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2023,3,27]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00498"},{"key":"e_1_3_2_1_2_1","volume-title":"Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence.","author":"Attenberg M","year":"2011","unstructured":"Josh\u00a0M Attenberg, Pagagiotis\u00a0G Ipeirotis, and Foster Provost. 2011. Beat the machine: Challenging workers to find the unknown unknowns. In Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_1_3_1","volume-title":"Recognition-by-components: a theory of human image understanding.Psychological review 94, 2","author":"Biederman Irving","year":"1987","unstructured":"Irving Biederman. 1987. Recognition-by-components: a theory of human image understanding.Psychological review 94, 2 (1987), 115."},{"volume-title":"Design things","author":"Binder Thomas","key":"e_1_3_2_1_4_1","unstructured":"Thomas Binder, Giorgio De\u00a0Michelis, Pelle Ehn, Giulio Jacucci, and Per Linde. 2011. Design things. MIT press."},{"volume-title":"Thematic analysis","author":"Braun Virginia","key":"e_1_3_2_1_5_1","unstructured":"Virginia Braun and Victoria Clarke. 2012. Thematic analysis.American Psychological Association."},{"key":"e_1_3_2_1_6_1","unstructured":"Anirban Chakraborty Manaar Alam Vishal Dey Anupam Chattopadhyay and Debdeep Mukhopadhyay. 2018. Adversarial attacks and defences: A survey. arXiv preprint arXiv:1810.00069(2018)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/tvcg.2020.2973258"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.5555\/1622737.1622744"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2756"},{"key":"e_1_3_2_1_10_1","volume-title":"ImageNet-trained CNNs are biased towards texture","author":"Geirhos Robert","year":"1811","unstructured":"Robert Geirhos, Patricia Rubisch, Claudio Michaelis, Matthias Bethge, Felix\u00a0A Wichmann, and Wieland Brendel. 2018. ImageNet-trained CNNs are biased towards texture; increasing shape bias improves accuracy and robustness. arXiv preprint arXiv:1811.12231(2018)."},{"key":"e_1_3_2_1_11_1","volume-title":"Jonas Rauber, Heiko\u00a0H Sch\u00fctt, Matthias Bethge, and Felix\u00a0A Wichmann.","author":"Geirhos Robert","year":"2018","unstructured":"Robert Geirhos, Carlos R\u00a0Medina Temme, Jonas Rauber, Heiko\u00a0H Sch\u00fctt, Matthias Bethge, and Felix\u00a0A Wichmann. 2018. Generalisation in humans and deep neural networks. arXiv preprint arXiv:1808.08750(2018)."},{"volume-title":"Discovery of grounded theory: Strategies for qualitative research","author":"Glaser G","key":"e_1_3_2_1_12_1","unstructured":"Barney\u00a0G Glaser and Anselm\u00a0L Strauss. 2017. Discovery of grounded theory: Strategies for qualitative research. Routledge."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Suchin Gururangan Swabha Swayamdipta Omer Levy Roy Schwartz Samuel\u00a0R Bowman and Noah\u00a0A Smith. 2018. Annotation artifacts in natural language inference data. arXiv preprint arXiv:1803.02324(2018).","DOI":"10.18653\/v1\/N18-2017"},{"key":"e_1_3_2_1_14_1","unstructured":"Moritz Hardt Eric Price and Nathan Srebro. 2016. Equality of opportunity in supervised learning. arXiv preprint arXiv:1610.02413(2016)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_16_1","volume-title":"Moving beyond P values: data analysis with estimation graphics. Nature methods 16, 7","author":"Ho Joses","year":"2019","unstructured":"Joses Ho, Tayfun Tumkaya, Sameer Aryal, Hyungwon Choi, and Adam Claridge-Chang. 2019. Moving beyond P values: data analysis with estimation graphics. Nature methods 16, 7 (2019), 565\u2013566."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/IC4.2009.4909197"},{"key":"e_1_3_2_1_19_1","unstructured":"Alina Kuznetsova Hassan Rom Neil Alldrin Jasper Uijlings Ivan Krasin Jordi Pont-Tuset Shahab Kamali Stefan Popov Matteo Malloci Alexander Kolesnikov 2018. The open images dataset v4: Unified image classification object detection and visual relationship detection at scale. arXiv preprint arXiv:1811.00982(2018)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"e_1_3_2_1_21_1","volume-title":"International Conference on Machine Learning. PMLR, 1078\u20131088","author":"Le\u00a0Bras Ronan","year":"2020","unstructured":"Ronan Le\u00a0Bras, Swabha Swayamdipta, Chandra Bhagavatula, Rowan Zellers, Matthew Peters, Ashish Sabharwal, and Yejin Choi. 2020. Adversarial filters of dataset biases. In International Conference on Machine Learning. PMLR, 1078\u20131088."},{"volume-title":"Proceedings of the 17th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"D.","key":"e_1_3_2_1_22_1","unstructured":"David\u00a0D. Lewis and William\u00a0A. Gale. 1994. A Sequential Algorithm for Training Text Classifiers. In Proceedings of the 17th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (Dublin, Ireland). 3\u201312."},{"key":"e_1_3_2_1_23_1","unstructured":"Lin Li and Michael Spratling. 2023. Data Augmentation Alone Can Improve Adversarial Training. arXiv preprint arXiv:2301.09879(2023)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380306"},{"key":"e_1_3_2_1_25_1","volume-title":"The ontology of concepts-abstract objects or mental representations?No\u00fbs 41, 4","author":"Margolis Eric","year":"2007","unstructured":"Eric Margolis and Stephen Laurence. 2007. The ontology of concepts-abstract objects or mental representations?No\u00fbs 41, 4 (2007), 561\u2013593."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359174"},{"key":"e_1_3_2_1_27_1","unstructured":"Rafael M\u00fcller Simon Kornblith and Geoffrey\u00a0E Hinton. 2019. When does label smoothing help?. In Advances in Neural Information Processing Systems. 4694\u20134703."},{"volume-title":"Human-in-the-Loop Machine Learning","author":"Munro Robert","key":"e_1_3_2_1_28_1","unstructured":"Robert Munro. 2021. Human-in-the-Loop Machine Learning. Manning Publications."},{"key":"e_1_3_2_1_29_1","volume-title":"MLOps: From Model-centric to Data-centric AI. https:\/\/www.deeplearning.ai\/wp-content\/uploads\/2021\/06\/MLOps-From-Model-centric-to-Data-centric-AI.pdf. Deeplearning.ai [Online","author":"Andrew NG.","year":"2021","unstructured":"Andrew NG. 2021. MLOps: From Model-centric to Data-centric AI. https:\/\/www.deeplearning.ai\/wp-content\/uploads\/2021\/06\/MLOps-From-Model-centric-to-Data-centric-AI.pdf. Deeplearning.ai [Online; posted: June-2021]."},{"key":"e_1_3_2_1_30_1","volume-title":"Proceedings of the 21st Conference on Geo-Information Science (AGILE","author":"Noorian Shahin\u00a0Sharifi","year":"2018","unstructured":"Shahin\u00a0Sharifi Noorian, Achilleas Psyllidis, and Alessandro Bozzon. 2018. A time-varying p-median model for location-allocation analysis. In Proceedings of the 21st Conference on Geo-Information Science (AGILE 2018). AGILE."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.3390\/app10228079"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2019.00971"},{"key":"e_1_3_2_1_33_1","unstructured":"Neoklis Polyzotis and Matei Zaharia. 2021. What can Data-Centric AI Learn from Data and ML Engineering?arXiv preprint arXiv:2112.06439(2021)."},{"key":"e_1_3_2_1_34_1","first-page":"65","article-title":"Conceptual combination: Models, theories and controversies","volume":"1","author":"Ran Bing","year":"2010","unstructured":"Bing Ran and P\u00a0Robert Duimering. 2010. Conceptual combination: Models, theories and controversies. International Journal of Cognitive Linguistics 1, 1(2010), 65\u201390.","journal-title":"International Journal of Cognitive Linguistics"},{"key":"e_1_3_2_1_35_1","unstructured":"Sylvestre-Alvise Rebuffi Sven Gowal Dan\u00a0Andrei Calian Florian Stimberg Olivia Wiles and Timothy Mann. 2021. Data Augmentation Can Improve Robustness. In Advances in Neural Information Processing Systems A.\u00a0Beygelzimer Y.\u00a0Dauphin P.\u00a0Liang and J.\u00a0Wortman Vaughan (Eds.). https:\/\/openreview.net\/forum?id=kgVJBBThdSZ"},{"key":"e_1_3_2_1_36_1","unstructured":"Amir Rosenfeld Richard Zemel and John\u00a0K Tsotsos. 2018. The elephant in the room. arXiv preprint arXiv:1808.03305(2018)."},{"key":"e_1_3_2_1_37_1","volume-title":"Data Cascades in High-Stakes AI. In proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 1\u201315","author":"Sambasivan Nithya","year":"2021","unstructured":"Nithya Sambasivan, Shivani Kapania, Hannah Highfill, Diana Akrong, Praveen Paritosh, and Lora\u00a0M Aroyo. 2021. \u201cEveryone wants to do the model work, not the data work\u201d: Data Cascades in High-Stakes AI. In proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 1\u201315."},{"key":"e_1_3_2_1_38_1","unstructured":"Burcu Sayin Fabio Casati Andrea Passerini Jie Yang and Xinyue Chen. 2022. Rethinking and Recomputing the Value of ML Models. arXiv preprint arXiv:2209.15157(2022)."},{"key":"e_1_3_2_1_39_1","unstructured":"Burcu Sayin Jie Yang Andrea Passerini and Fabio Casati. 2021. The science of rejection: a research area for human computation. arXiv preprint arXiv:2111.06736(2021)."},{"key":"e_1_3_2_1_40_1","unstructured":"Shreya Shankar Yoni Halpern Eric Breck James Atwood Jimbo Wilson and D Sculley. 2017. No classification without representation: Assessing geodiversity issues in open data sets for the developing world. arXiv preprint arXiv:1711.08536(2017)."},{"key":"e_1_3_2_1_41_1","volume-title":"International Conference on Machine Learning. PMLR, 8634\u20138644","author":"Shankar Vaishaal","year":"2020","unstructured":"Vaishaal Shankar, Rebecca Roelofs, Horia Mania, Alex Fang, Benjamin Recht, and Ludwig Schmidt. 2020. Evaluating machine accuracy on imagenet. In International Conference on Machine Learning. PMLR, 8634\u20138644."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512040"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/570907.570945"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-019-0197-0"},{"key":"e_1_3_2_1_45_1","volume-title":"3rd International Conference on Learning Representations, ICLR","author":"Simonyan Karen","year":"2015","unstructured":"Karen Simonyan and Andrew Zisserman. 2015. Very Deep Convolutional Networks for Large-Scale Image Recognition. In 3rd International Conference on Learning Representations, ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings, Yoshua Bengio and Yann LeCun (Eds.). http:\/\/arxiv.org\/abs\/1409.1556"},{"key":"e_1_3_2_1_46_1","unstructured":"Daniel Smilkov Nikhil Thorat Been Kim Fernanda Vi\u00e9gas and Martin Wattenberg. 2017. Smoothgrad: removing noise by adding noise. arXiv preprint arXiv:1706.03825(2017)."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-01231-1_31"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"e_1_3_2_1_49_1","volume-title":"2nd International Conference on Learning Representations, ICLR","author":"Szegedy Christian","year":"2014","unstructured":"Christian Szegedy, Wojciech Zaremba, Ilya Sutskever, Joan Bruna, Dumitru Erhan, Ian Goodfellow, and Rob Fergus. 2014. Intriguing properties of neural networks. In 2nd International Conference on Learning Representations, ICLR 2014."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST50239.2020.00006"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313599"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3375709"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST50239.2020.00007"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2021.3050071"},{"key":"e_1_3_2_1_55_1","volume-title":"Neural Motifs: Scene Graph Parsing with Global Context. In Conference on Computer Vision and Pattern Recognition.","author":"Zellers Rowan","year":"2018","unstructured":"Rowan Zellers, Mark Yatskar, Sam Thomson, and Yejin Choi. 2018. Neural Motifs: Scene Graph Parsing with Global Context. In Conference on Computer Vision and Pattern Recognition."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359158"}],"event":{"name":"IUI '23: 28th International Conference on Intelligent User Interfaces","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Sydney NSW Australia","acronym":"IUI '23"},"container-title":["Proceedings of the 28th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3581641.3584096","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3581641.3584096","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:21Z","timestamp":1750178181000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3581641.3584096"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,3,27]]},"references-count":56,"alternative-id":["10.1145\/3581641.3584096","10.1145\/3581641"],"URL":"https:\/\/doi.org\/10.1145\/3581641.3584096","relation":{},"subject":[],"published":{"date-parts":[[2023,3,27]]},"assertion":[{"value":"2023-03-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}