{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T13:09:40Z","timestamp":1765544980448,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,21]],"date-time":"2024-10-21T00:00:00Z","timestamp":1729468800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,21]]},"DOI":"10.1145\/3627673.3679777","type":"proceedings-article","created":{"date-parts":[[2024,10,20]],"date-time":"2024-10-20T19:34:21Z","timestamp":1729452861000},"page":"2097-2107","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["XCrowd: Combining Explainability and Crowdsourcing to Diagnose Models in Relation Extraction"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7108-9917","authenticated-orcid":false,"given":"Alisa","family":"Smirnova","sequence":"first","affiliation":[{"name":"University of Fribourg, Fribourg, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0350-0313","authenticated-orcid":false,"given":"Jie","family":"Yang","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2588-4212","authenticated-orcid":false,"given":"Philippe","family":"Cudre-Mauroux","sequence":"additional","affiliation":[{"name":"University of Fribourg, Fribourg, Switzerland"}]}],"member":"320","published-online":{"date-parts":[[2024,10,21]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.142"},{"key":"e_1_3_2_1_2_1","volume-title":"Knowledge of Knowledge: Exploring Known-Unknowns Uncertainty with Large Language Models. arXiv preprint arXiv:2305.13712","author":"Amayuelas Alfonso","year":"2023","unstructured":"Alfonso Amayuelas, Liangming Pan, Wenhu Chen, and William Wang. 2023. Knowledge of Knowledge: Exploring Known-Unknowns Uncertainty with Large Language Models. arXiv preprint arXiv:2305.13712 (2023)."},{"key":"e_1_3_2_1_3_1","series-title":"AAAI Workshops","volume-title":"Beat the machine: Challenging workers to find the unknown unknowns, in 'Human Computation","author":"Attenberg J","unstructured":"J Attenberg, PG Ipeirotis, and FJ Provost. 2011. Beat the machine: Challenging workers to find the unknown unknowns, in 'Human Computation', Vol. WS-11--11 of AAAI Workshops, AAAI."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v10i1.21985"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-emnlp.204"},{"key":"e_1_3_2_1_6_1","volume-title":"Daniele Theseider Dupr\u00e9, and Pietro Torasso","author":"Console Luca","year":"1989","unstructured":"Luca Console, Daniele Theseider Dupr\u00e9, and Pietro Torasso. 1989. A Theory of Diagnosis for Incomplete Causal Models.. In IJCAI. 1311--1317."},{"key":"e_1_3_2_1_7_1","volume-title":"International Conference on Artificial Intelligence and Statistics. PMLR, 2845--2853","author":"Deng Zhun","year":"2021","unstructured":"Zhun Deng, Linjun Zhang, Amirata Ghorbani, and James Zou. 2021. Improving adversarial robustness via unlabeled out-of-domain data. In International Conference on Artificial Intelligence and Statistics. PMLR, 2845--2853."},{"key":"e_1_3_2_1_8_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v38i3.2756"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488421"},{"key":"e_1_3_2_1_11_1","volume-title":"A systematic review of robustness in deep learning for computer vision: Mind the gap? arXiv preprint arXiv:2112.00639","author":"Drenkow Nathan","year":"2021","unstructured":"Nathan Drenkow, Numair Sani, Ilya Shpitser, and Mathias Unberath. 2021. A systematic review of robustness in deep learning for computer vision: Mind the gap? arXiv preprint arXiv:2112.00639 (2021)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2020.3031769"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW50498.2020.00398"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.491"},{"key":"e_1_3_2_1_15_1","volume-title":"When Can Models Learn From Explanations? A Formal Framework for Understanding the Roles of Explanation Data. CoRR abs\/2102.02201","author":"Hase Peter","year":"2021","unstructured":"Peter Hase and Mohit Bansal. 2021. When Can Models Learn From Explanations? A Formal Framework for Understanding the Roles of Explanation Data. CoRR abs\/2102.02201 (2021). arXiv:2102.02201 https:\/\/arxiv.org\/abs\/2102.02201"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866701"},{"key":"e_1_3_2_1_17_1","volume-title":"Robust convolutional neural networks under adversarial noise. arXiv preprint arXiv:1511.06306","author":"Jin Jonghoon","year":"2015","unstructured":"Jonghoon Jin, Aysegul Dundar, and Eugenio Culurciello. 2015. Robust convolutional neural networks under adversarial noise. arXiv preprint arXiv:1511.06306 (2015)."},{"key":"e_1_3_2_1_18_1","volume-title":"SpanBERT: Improving Pre-training by Representing and Predicting Spans. arXiv preprint arXiv:1907.10529","author":"Joshi Mandar","year":"2019","unstructured":"Mandar Joshi, Danqi Chen, Yinhan Liu, Daniel S. Weld, Luke Zettlemoyer, and Omer Levy. 2019. SpanBERT: Improving Pre-training by Representing and Predicting Spans. arXiv preprint arXiv:1907.10529 (2019)."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Pavan Kapanipathi Ibrahim Abdelaziz Srinivas Ravishankar Salim Roukos Alexander Gray Ramon Astudillo Maria Chang Cristina Cornelio Saswati Dana Achille Fokoue et al. 2020. Leveraging abstract meaning representation for knowledge base question answering. arXiv preprint arXiv:2012.01707 (2020).","DOI":"10.18653\/v1\/2021.findings-acl.339"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Edith Law and Luis Von Ahn. 2011. Human computation. (2011).","DOI":"10.1007\/978-3-031-01555-7"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00440"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380306"},{"key":"e_1_3_2_1_25_1","volume-title":"Roberta: A robustly optimized bert pretraining approach. arXiv preprint arXiv:1907.11692","author":"Liu Yinhan","year":"2019","unstructured":"Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. 2019. Roberta: A robustly optimized bert pretraining approach. arXiv preprint arXiv:1907.11692 (2019)."},{"key":"e_1_3_2_1_26_1","first-page":"I","article-title":"A Unified Approach to Interpreting Model Predictions","volume":"30","author":"Lundberg Scott M","year":"2017","unstructured":"Scott M Lundberg and Su-In Lee. 2017. A Unified Approach to Interpreting Model Predictions. In Advances in Neural Information Processing Systems 30, I. Guyon, U. V. Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (Eds.). Curran Associates, Inc., 4765--4774. http:\/\/papers.nips.cc\/paper\/7062-a-unified-approach-to-interpreting-model-predictions.pdf","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13287"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866714"},{"key":"e_1_3_2_1_29_1","volume-title":"End-to-end relation extraction using lstms on sequences and tree structures. arXiv preprint arXiv:1601.00770","author":"Miwa Makoto","year":"2016","unstructured":"Makoto Miwa and Mohit Bansal. 2016. End-to-end relation extraction using lstms on sequences and tree structures. arXiv preprint arXiv:1601.00770 (2016)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/W15-1506"},{"key":"e_1_3_2_1_31_1","volume-title":"Manning","author":"Pennington Jeffrey","year":"2014","unstructured":"Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. GloVe: Global Vectors for Word Representation. In Empirical Methods in Natural Language Processing (EMNLP). 1532--1543. http:\/\/www.aclweb.org\/anthology\/D14--1162"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the Eighth Conference on Computational Natural Language Learning, CoNLL 2004","author":"Roth Dan","year":"2004","unstructured":"Dan Roth and Wen-tau Yih. 2004. A Linear Programming Formulation for Global Inference in Natural Language Tasks. In Proceedings of the Eighth Conference on Computational Natural Language Learning, CoNLL 2004, Held in cooperation with HLT-NAACL 2004, Boston, Massachusetts, USA, May 6--7, 2004, Hwee Tou Ng and Ellen Riloff (Eds.). ACL, 1--8. https:\/\/aclanthology.org\/W04--2401\/"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the ACM Web Conference","author":"Noorian Shahin Sharifi","year":"2022","unstructured":"Shahin Sharifi Noorian, Sihang Qiu, Ujwal Gadiraju, Jie Yang, and Alessandro Bozzon. 2022. What Should You Know? A Human-In-the-Loop Approach to Unknown Unknowns Characterization in Image Recognition. In Proceedings of the ACM Web Conference 2022. 882--892."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-58526-6_24"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2022.3199570"},{"key":"e_1_3_2_1_37_1","volume-title":"Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199","author":"Szegedy Christian","year":"2013","unstructured":"Christian Szegedy, Wojciech Zaremba, Ilya Sutskever, Joan Bruna, Dumitru Erhan, Ian Goodfellow, and Rob Fergus. 2013. Intriguing properties of neural networks. arXiv preprint arXiv:1312.6199 (2013)."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00463"},{"key":"e_1_3_2_1_39_1","volume-title":"AI Robustness: a Human-Centered Perspective on Technological Challenges and Opportunities. arXiv preprint arXiv:2210.08906","author":"Tocchetti Andrea","year":"2022","unstructured":"Andrea Tocchetti, Lorenzo Corti, Agathe Balayn, Mireia Yurrita, Philip Lippmann, Marco Brambilla, and Jie Yang. 2022. AI Robustness: a Human-Centered Perspective on Technological Challenges and Opportunities. arXiv preprint arXiv:2210.08906 (2022)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.5555\/3122009.3242050"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_42_1","volume-title":"8th International Conference on Learning Representations, ICLR 2020","author":"Wang Ziqi","year":"2020","unstructured":"Ziqi Wang, Yujia Qin, Wenxuan Zhou, Jun Yan, Qinyuan Ye, Leonardo Neves, Zhiyuan Liu, and Xiang Ren. 2020. Learning from Explanations with Neural Execution Tree. In 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26--30, 2020. OpenReview.net. https:\/\/openreview.net\/forum?id=rJlUt0EYwS"},{"volume-title":"Making things happen: A theory of causal explanation","author":"Woodward James","key":"e_1_3_2_1_43_1","unstructured":"James Woodward. 2005. Making things happen: A theory of causal explanation. Oxford university press."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357384.3358119"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313599"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D17-1004"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.14778\/3055540.3055547"},{"key":"e_1_3_2_1_48_1","volume-title":"An improved baseline for sentence-level relation extraction. arXiv preprint arXiv:2102.01373","author":"Zhou Wenxuan","year":"2021","unstructured":"Wenxuan Zhou and Muhao Chen. 2021. An improved baseline for sentence-level relation extraction. arXiv preprint arXiv:2102.01373 (2021)."}],"event":{"name":"CIKM '24: The 33rd ACM International Conference on Information and Knowledge Management","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Boise ID USA","acronym":"CIKM '24"},"container-title":["Proceedings of the 33rd ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627673.3679777","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3627673.3679777","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:58:28Z","timestamp":1750294708000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627673.3679777"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,21]]},"references-count":48,"alternative-id":["10.1145\/3627673.3679777","10.1145\/3627673"],"URL":"https:\/\/doi.org\/10.1145\/3627673.3679777","relation":{},"subject":[],"published":{"date-parts":[[2024,10,21]]},"assertion":[{"value":"2024-10-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}