{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,30]],"date-time":"2026-09-30T00:36:28Z","timestamp":1790728588079,"version":"4.1.0"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T00:00:00Z","timestamp":1650844800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,4,25]]},"DOI":"10.1145\/3485447.3512040","type":"proceedings-article","created":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T05:13:07Z","timestamp":1650863587000},"page":"882-892","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["What Should You Know? A Human-In-the-Loop Approach to Unknown Unknowns Characterization in Image Recognition"],"prefix":"10.1145","author":[{"given":"Shahin","family":"Sharifi Noorian","sequence":"first","affiliation":[{"name":"Delft University of Technology, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sihang","family":"Qiu","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ujwal","family":"Gadiraju","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jie","family":"Yang","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alessandro","family":"Bozzon","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,4,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/66.2.229"},{"key":"e_1_3_2_1_2_1","volume-title":"Detecting anomalies in cross-classified streams: a bayesian approach. Knowledge and information systems 11, 1","author":"Agarwal Deepak","year":"2007","unstructured":"Deepak Agarwal. 2007. Detecting anomalies in cross-classified streams: a bayesian approach. Knowledge and information systems 11, 1 (2007), 29\u201344."},{"key":"e_1_3_2_1_3_1","volume-title":"International Conference on Machine Learning. PMLR, 80\u201388","author":"Allamanis Miltiadis","year":"2017","unstructured":"Miltiadis Allamanis, Pankajan Chanthirasegaran, Pushmeet Kohli, and Charles Sutton. 2017. Learning continuous semantic representations of symbolic expressions. In International Conference on Machine Learning. PMLR, 80\u201388."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702509"},{"key":"e_1_3_2_1_5_1","volume-title":"Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence.","author":"Attenberg M","year":"2011","unstructured":"Josh\u00a0M Attenberg, Pagagiotis\u00a0G Ipeirotis, and Foster Provost. 2011. Beat the machine: Challenging workers to find the unknown unknowns. In Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450069"},{"key":"e_1_3_2_1_7_1","article-title":"Online Choice of Active Learning Algorithms","author":"Baram Yoram","year":"2004","unstructured":"Yoram Baram, Ran El-Yaniv, and Kobi Luz. 2004. Online Choice of Active Learning Algorithms. Journal of Machine Learning Research 5 (Dec. 2004), 255\u2013291.","journal-title":"Journal of Machine Learning Research 5"},{"key":"e_1_3_2_1_8_1","first-page":"15","article-title":"Outlier detection: A survey","volume":"14","author":"Chandola Varun","year":"2007","unstructured":"Varun Chandola, Arindam Banerjee, and Vipin Kumar. 2007. Outlier detection: A survey. Comput. Surveys 14(2007), 15.","journal-title":"Comput. Surveys"},{"key":"e_1_3_2_1_9_1","volume-title":"Can Active Learning Experience Be Transferred?IEEE 16th International Conference on Data Mining","author":"Chu Hong-Min","year":"2016","unstructured":"Hong-Min Chu and Hsuan-Tien Lin. 2016. Can Active Learning Experience Be Transferred?IEEE 16th International Conference on Data Mining (2016), 841\u2013846."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359786"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v8i1.7462"},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the Seventeenth International Conference on Machine Learning. 255\u2013262","author":"Eskin Eleazar","year":"2000","unstructured":"Eleazar Eskin. 2000. Anomaly Detection over Noisy Data using Learned Probability Distributions. In Proceedings of the Seventeenth International Conference on Machine Learning. 255\u2013262."},{"key":"e_1_3_2_1_14_1","volume-title":"Applications of data mining in computer security","author":"Eskin Eleazar","unstructured":"Eleazar Eskin, Andrew Arnold, Michael Prerau, Leonid Portnoy, and Sal Stolfo. 2002. A geometric framework for unsupervised anomaly detection. In Applications of data mining in computer security. Springer, 77\u2013101."},{"key":"e_1_3_2_1_15_1","volume-title":"Adaptive fraud detection. Data mining and knowledge discovery 1, 3","author":"Fawcett Tom","year":"1997","unstructured":"Tom Fawcett and Foster Provost. 1997. Adaptive fraud detection. Data mining and knowledge discovery 1, 3 (1997), 291\u2013316."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.3389\/978-2-88919-940-2"},{"key":"e_1_3_2_1_17_1","volume-title":"2020 Crowd Science Workshop: Remoteness, Fairness, and Mechanisms as Challenges of Data Supply by Humans for Automation. 7\u201313","author":"Gadiraju Ujwal","year":"2020","unstructured":"Ujwal Gadiraju and Jie Yang. 2020. What can crowd computing do for the next generation of AI systems?. In 2020 Crowd Science Workshop: Remoteness, Fairness, and Mechanisms as Challenges of Data Supply by Humans for Automation. 7\u201313."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2916887"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_20_1","volume-title":"Revealing the multidimensional mental representations of natural objects underlying human similarity judgements. Nature Human Behaviour 4, 11","author":"Hebart N","year":"2020","unstructured":"Martin\u00a0N Hebart, Charles\u00a0Y Zheng, Francisco Pereira, and Chris\u00a0I Baker. 2020. Revealing the multidimensional mental representations of natural objects underlying human similarity judgements. Nature Human Behaviour 4, 11 (2020)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.5555\/2886521.2886691"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380063"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"e_1_3_2_1_24_1","volume-title":"Semi-Supervised Classification with Graph Convolutional Networks. In Intl. Conf. on Learning Representations.","author":"N.","unstructured":"Thomas\u00a0N. Kipf and Max Welling. 2017. Semi-Supervised Classification with Graph Convolutional Networks. In Intl. Conf. on Learning Representations."},{"key":"e_1_3_2_1_25_1","volume-title":"A revision of Bloom\u2019s taxonomy: An overview. Theory into practice 41, 4","author":"Krathwohl R","year":"2002","unstructured":"David\u00a0R Krathwohl. 2002. A revision of Bloom\u2019s taxonomy: An overview. Theory into practice 41, 4 (2002), 212\u2013218."},{"key":"e_1_3_2_1_26_1","volume-title":"Imagenet classification with deep convolutional neural networks. Advances in neural information processing systems 25","author":"Krizhevsky Alex","year":"2012","unstructured":"Alex Krizhevsky, Ilya Sutskever, and Geoffrey\u00a0E Hinton. 2012. Imagenet classification with deep convolutional neural networks. Advances in neural information processing systems 25 (2012), 1097\u20131105."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of the 17th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. 3\u201312","author":"D.","unstructured":"David\u00a0D. Lewis and William\u00a0A. Gale. 1994. A Sequential Algorithm for Training Text Classifiers. In Proceedings of the 17th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. 3\u201312."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380306"},{"key":"e_1_3_2_1_30_1","volume-title":"The ontology of concepts-abstract objects or mental representations?No\u00fbs 41, 4","author":"Margolis Eric","year":"2007","unstructured":"Eric Margolis and Stephen Laurence. 2007. The ontology of concepts-abstract objects or mental representations?No\u00fbs 41, 4 (2007), 561\u2013593."},{"key":"e_1_3_2_1_31_1","volume-title":"Jeremy\u00a0E Block, and Eric\u00a0D Ragan","author":"Mohseni Sina","year":"2021","unstructured":"Sina Mohseni, Jeremy\u00a0E Block, and Eric\u00a0D Ragan. 2021. Quantitative Evaluation of Machine Learning Explanations: A Human-Grounded Benchmark. (2021)."},{"key":"e_1_3_2_1_32_1","volume-title":"Universal Readout for Graph Convolutional Neural Networks. In 2019 International Joint Conference on Neural Networks. 1\u20137.","author":"Navarin Nicol\u00f2","year":"2019","unstructured":"Nicol\u00f2 Navarin, Dinh\u00a0Van Tran, and Alessandro Sperduti. 2019. Universal Readout for Graph Convolutional Neural Networks. In 2019 International Joint Conference on Neural Networks. 1\u20137."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10633"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Jeffrey Pennington Richard Socher and Christopher\u00a0D. Manning. 2014. GloVe: Global Vectors for Word Representation. In Empirical Methods in Natural Language Processing. 1532\u20131543.","DOI":"10.3115\/v1\/D14-1162"},{"key":"e_1_3_2_1_35_1","volume-title":"Machine behaviour. Nature 568, 7753","author":"Rahwan Iyad","year":"2019","unstructured":"Iyad Rahwan, Manuel Cebrian, Nick Obradovich, Josh Bongard, Jean-Fran\u00e7ois Bonnefon, Cynthia Breazeal, Jacob\u00a0W Crandall, Nicholas\u00a0A Christakis, Iain\u00a0D Couzin, Matthew\u00a0O Jackson, 2019. Machine behaviour. Nature 568, 7753 (2019), 477\u2013486."},{"key":"e_1_3_2_1_36_1","first-page":"65","article-title":"Conceptual combination: Models, theories and controversies","volume":"1","author":"Ran Bing","year":"2010","unstructured":"Bing Ran and P\u00a0Robert Duimering. 2010. Conceptual combination: Models, theories and controversies. International Journal of Cognitive Linguistics 1, 1(2010), 65\u201390.","journal-title":"International Journal of Cognitive Linguistics"},{"key":"e_1_3_2_1_37_1","volume-title":"Learning from crowds.Journal of Machine Learning Research 11, 4","author":"Raykar C","year":"2010","unstructured":"Vikas\u00a0C Raykar, Shipeng Yu, Linda\u00a0H Zhao, Gerardo\u00a0Hermosillo Valadez, Charles Florin, Luca Bogoni, and Linda Moy. 2010. Learning from crowds.Journal of Machine Learning Research 11, 4 (2010)."},{"key":"e_1_3_2_1_38_1","volume-title":"Faster r-cnn: Towards real-time object detection with region proposal networks","author":"Ren Shaoqing","year":"2016","unstructured":"Shaoqing Ren, Kaiming He, Ross Girshick, and Jian Sun. 2016. Faster r-cnn: Towards real-time object detection with region proposal networks. IEEE transactions on pattern analysis and machine intelligence 39, 6(2016), 1137\u20131149."},{"key":"e_1_3_2_1_39_1","unstructured":"Nicholas Roy and Andrew McCallum. 2001. Toward Optimal Active Learning Through Sampling Estimation of Error Reduction. In ICML. 894\u2013905."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6399"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Nithya Sambasivan Shivani Kapania Hannah Highfill Diana Akrong Praveen\u00a0Kumar Paritosh and Lora\u00a0Mois Aroyo. 2021. \u201dEveryone wants to do the model work not the data work\u201d: Data Cascades in High-Stakes AI. (2021).","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Burr Settles. 2012. Active learning. Synthesis lectures on artificial intelligence and machine learning 6 1(2012) 1\u2013114.","DOI":"10.2200\/S00429ED1V01Y201207AIM018"},{"key":"e_1_3_2_1_43_1","volume-title":"Proceedings of the Fifth Annual Workshop on Computational Learning Theory","author":"Seung S.","unstructured":"H.\u00a0S. Seung, M. Opper, and H. Sompolinsky. 1992. Query by Committee. In Proceedings of the Fifth Annual Workshop on Computational Learning Theory (Pittsburgh, Pennsylvania, USA). 287\u2013294."},{"key":"e_1_3_2_1_44_1","volume-title":"Models of thought. Vol.\u00a0352","author":"Simon A.","unstructured":"Herbert\u00a0A. Simon. 1979. Models of thought. Vol.\u00a0352. Yale university press."},{"key":"e_1_3_2_1_45_1","unstructured":"Karen Simonyan Andrea Vedaldi and Andrew Zisserman. 2013. Deep inside convolutional networks: Visualising image classification models and saliency maps. arXiv preprint arXiv:1312.6034(2013)."},{"key":"e_1_3_2_1_46_1","volume-title":"Smoothgrad: removing noise by adding noise. arXiv:1706.03825","author":"Smilkov Daniel","year":"2017","unstructured":"Daniel Smilkov, Nikhil Thorat, Been Kim, Fernanda Vi\u00e9gas, and Martin Wattenberg. 2017. Smoothgrad: removing noise by adding noise. arXiv:1706.03825 (2017)."},{"key":"e_1_3_2_1_47_1","volume-title":"International Conference on Learning Representations.","author":"Sun Fan-Yun","year":"2019","unstructured":"Fan-Yun Sun, Jordan Hoffman, Vikas Verma, and Jian Tang. 2019. InfoGraph: Unsupervised and Semi-supervised Graph-Level Representation Learning via Mutual Information Maximization. In International Conference on Learning Representations."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.11231"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5274"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3219819.3220070"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"crossref","unstructured":"Jie Yang Alisa Smirnova Dingqi Yang Gianluca Demartini Yuan Lu and Philippe Cudr\u00e9-Mauroux. 2019. Scalpel-cd: leveraging crowdsourcing and deep probabilistic modeling for debugging noisy training data. In WWW. 2158\u20132168.","DOI":"10.1145\/3308558.3313599"},{"key":"e_1_3_2_1_52_1","unstructured":"Rex Ying Jiaxuan You Christopher Morris Xiang Ren William\u00a0L Hamilton and Jure Leskovec. 2018. Hierarchical graph representation learning with differentiable pooling. arXiv preprint arXiv:1806.08804(2018)."},{"key":"e_1_3_2_1_53_1","volume-title":"Neural Motifs: Scene Graph Parsing with Global Context. In Conference on Computer Vision and Pattern Recognition.","author":"Zellers Rowan","year":"2018","unstructured":"Rowan Zellers, Mark Yatskar, Sam Thomson, and Yejin Choi. 2018. Neural Motifs: Scene Graph Parsing with Global Context. In Conference on Computer Vision and Pattern Recognition."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359158"},{"key":"e_1_3_2_1_55_1","unstructured":"Bolei Zhou Agata Lapedriza Jianxiong Xiao Antonio Torralba and Aude Oliva. 2014. Learning deep features for scene recognition using places database. (2014)."}],"event":{"name":"WWW '22: The ACM Web Conference 2022","location":"Virtual Event, Lyon France","acronym":"WWW '22","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM Web Conference 2022"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485447.3512040","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3485447.3512040","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:06Z","timestamp":1750188606000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485447.3512040"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,25]]},"references-count":55,"alternative-id":["10.1145\/3485447.3512040","10.1145\/3485447"],"URL":"https:\/\/doi.org\/10.1145\/3485447.3512040","relation":{},"subject":[],"published":{"date-parts":[[2022,4,25]]},"assertion":[{"value":"2022-04-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}