{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,14]],"date-time":"2026-07-14T22:52:12Z","timestamp":1784069532217,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,10]],"date-time":"2022-06-10T00:00:00Z","timestamp":1654819200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100006785","name":"Google","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006785","id-type":"DOI","asserted-by":"publisher"}]},{"name":"CRD"},{"name":"Amazon"},{"name":"NSERC"},{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["1845638,2008295,2106197,2103794"],"award-info":[{"award-number":["1845638,2008295,2106197,2103794"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004489","name":"Mitacs","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004489","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,10]]},"DOI":"10.1145\/3514221.3517849","type":"proceedings-article","created":{"date-parts":[[2022,6,12]],"date-time":"2022-06-12T02:33:49Z","timestamp":1655001229000},"page":"369-383","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Complaint-Driven Training Data Debugging at Interactive Speeds"],"prefix":"10.1145","author":[{"given":"Lampros","family":"Flokas","sequence":"first","affiliation":[{"name":"Columbia University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weiyuan","family":"Wu","sequence":"additional","affiliation":[{"name":"Simon Fraser University, Burnaby, BC, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yejia","family":"Liu","sequence":"additional","affiliation":[{"name":"Simon Fraser University, Burnaby, BC, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiannan","family":"Wang","sequence":"additional","affiliation":[{"name":"Simon Fraser University, Burnaby, BC, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nakul","family":"Verma","sequence":"additional","affiliation":[{"name":"Columbia University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eugene","family":"Wu","sequence":"additional","affiliation":[{"name":"Columbia University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,6,11]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.14778\/3297753.3297761"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1989284.1989302"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2645710.2645741"},{"key":"e_1_3_2_1_4_1","volume-title":"Steven Euijong Whang, and Martin Zinkevich","author":"Breck Eric","year":"2019","unstructured":"Eric Breck, Neoklis Polyzotis, Sudip Roy, Steven Euijong Whang, and Martin Zinkevich. 2019. Data Validation for Machine Learning. https:\/\/mlsys.org\/Conferences\/2019\/doc\/2019\/167.pdf"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1984.10478080"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"George Corliss Christ\u00e8le Faure Andreas Griewank Laurent Hasco\u00ebt and Uwe Naumann (Eds.). 2002. Differentiation Methods for Industrial Strength Problems .Springer New York New York NY. https:\/\/doi.org\/10.1007\/978--1--4613-0075--5_1","DOI":"10.1007\/978--1--4613-0075--5_1"},{"key":"e_1_3_2_1_7_1","volume-title":"Advances in Neural Information Processing Systems","author":"Dong Zhen","year":"1851","unstructured":"Zhen Dong, Zhewei Yao, Daiyaan Arfeen, Amir Gholami, Michael W Mahoney, and Kurt Keutzer. 2020. HAWQ-V2: Hessian Aware trace-Weighted Quantization of Neural Networks. In Advances in Neural Information Processing Systems, H. Larochelle, M. Ranzato, R. Hadsell, M. F. Balcan, and H. Lin (Eds.), Vol. 33. Curran Associates, Inc., Redhook, NY, USA, 18518--18529. https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/d77c703536718b95308130ff2e5cf9ee-Paper.pdf"},{"key":"e_1_3_2_1_8_1","unstructured":"Dheeru Dua and Casey Graff. 2017. UCI Machine Learning Repository. http:\/\/archive.ics.uci.edu\/ml"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02288367"},{"key":"e_1_3_2_1_10_1","volume-title":"https:\/\/onnx.ai\/. [Online","author":"Network Exchange Open Neural","year":"2020","unstructured":"Open Neural Network Exchange. 2019. ONNX. https:\/\/onnx.ai\/. [Online; accessed 1-December-2020]."},{"key":"e_1_3_2_1_11_1","volume-title":"Trainable Neural Networks. In 7th International Conference on Learning Representations, ICLR 2019","author":"Frankle Jonathan","year":"2019","unstructured":"Jonathan Frankle and Michael Carbin. 2019. The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks. In 7th International Conference on Learning Representations, ICLR 2019, New Orleans, LA, USA, May 6--9, 2019. OpenReview.net."},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"2241","author":"Ghorbani Behrooz","year":"2019","unstructured":"Behrooz Ghorbani, Shankar Krishnan, and Ying Xiao. 2019. An Investigation into Neural Net Optimization via Hessian Eigenvalue Density. In Proceedings of the 36th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 97), Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.). PMLR, 2232--2241. https:\/\/proceedings.mlr.press\/v97\/ghorbani19b.html"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1037\/1076-898X.6.4.322"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1265530.1265535"},{"key":"e_1_3_2_1_15_1","volume-title":"Gradient Descent Happens in a Tiny Subspace. CoRR","author":"Gur-Ari Guy","year":"2018","unstructured":"Guy Gur-Ari, Daniel A. Roberts, and Ethan Dyer. 2018. Gradient Descent Happens in a Tiny Subspace. CoRR, Vol. abs\/1812.04754 (2018). arxiv: 1812.04754 http:\/\/arxiv.org\/abs\/1812.04754"},{"key":"e_1_3_2_1_16_1","volume-title":"Advances in Neural Information Processing Systems, H. Wallach, H. Larochelle, A. Beygelzimer, F. dtextquotesingle Alch\u00e9-Buc","author":"Hara Satoshi","year":"2019","unstructured":"Satoshi Hara, Atsushi Nitanda, and Takanori Maehara. 2019. Data Cleansing for Models Trained with SGD. In Advances in Neural Information Processing Systems, H. Wallach, H. Larochelle, A. Beygelzimer, F. dtextquotesingle Alch\u00e9-Buc, E. Fox, and R. Garnett (Eds.), Vol. 32. Curran Associates, Inc., Redhook, NY, USA. https:\/\/proceedings.neurips.cc\/paper\/2019\/file\/5f14615696649541a025d3d0f8e0447f-Paper.pdf"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753357"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.14778\/2367502.2367510"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.6028\/jres.049.044"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of The 24th International Conference on Artificial Intelligence and Statistics (Proceedings of Machine Learning Research","volume":"2016","author":"Izzo Zachary","year":"2021","unstructured":"Zachary Izzo, Mary Anne Smart, Kamalika Chaudhuri, and James Zou. 2021. Approximate Data Deletion from Machine Learning Models. In Proceedings of The 24th International Conference on Artificial Intelligence and Statistics (Proceedings of Machine Learning Research, Vol. 130), Arindam Banerjee and Kenji Fukumizu (Eds.). PMLR, 2008--2016. https:\/\/proceedings.mlr.press\/v130\/izzo21a.html"},{"key":"e_1_3_2_1_21_1","volume-title":"JAX reference documentation - JAX documentation. https:\/\/jax.readthedocs.io\/en\/latest\/. [Online","author":"JAX.","year":"2020","unstructured":"JAX. 2020. JAX reference documentation - JAX documentation. https:\/\/jax.readthedocs.io\/en\/latest\/. [Online; accessed 1-December-2020]."},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"1894","author":"Koh Pang Wei","year":"2017","unstructured":"Pang Wei Koh and Percy Liang. 2017. Understanding Black-box Predictions via Influence Functions. In Proceedings of the 34th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 70), Doina Precup and Yee Whye Teh (Eds.). PMLR, 1885--1894. https:\/\/proceedings.mlr.press\/v70\/koh17a.html"},{"key":"e_1_3_2_1_23_1","volume-title":"Advances in Neural Information Processing Systems, H. Wallach, H. Larochelle, A. Beygelzimer, F. dtextquotesingle Alch\u00e9-Buc","author":"Koh Pang Wei W","year":"2019","unstructured":"Pang Wei W Koh, Kai-Siang Ang, Hubert Teo, and Percy S Liang. 2019. On the Accuracy of Influence Functions for Measuring Group Effects. In Advances in Neural Information Processing Systems, H. Wallach, H. Larochelle, A. Beygelzimer, F. dtextquotesingle Alch\u00e9-Buc, E. Fox, and R. Garnett (Eds.), Vol. 32. Curran Associates, Inc., Redhook, NY, USA. https:\/\/proceedings.neurips.cc\/paper\/2019\/file\/a78482ce76496fcf49085f2190e675b4-Paper.pdf"},{"key":"e_1_3_2_1_24_1","unstructured":"Alex Krizhevsky. 2009. Learning Multiple Layers of Features from Tiny Images. (2009)."},{"key":"e_1_3_2_1_25_1","volume-title":"An iteration method for the solution of the eigenvalue problem of linear differential and integral operators .United States Governm","author":"Lanczos Cornelius","unstructured":"Cornelius Lanczos. 1950. An iteration method for the solution of the eigenvalue problem of linear differential and integral operators .United States Governm. Press Office Los Angeles, CA."},{"key":"e_1_3_2_1_26_1","volume-title":"Burges","author":"LeCun Yann","year":"2010","unstructured":"Yann LeCun, Corinna Cortes, and Christopher J.C. Burges. 2010. MNIST handwritten digit database. http:\/\/yann.lecun.com\/exdb\/mnist\/."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE51399.2021.00009"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346452"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2010.177"},{"key":"e_1_3_2_1_30_1","volume-title":"Introduction to BigQuery ML. https:\/\/cloud.google.com\/bigquery-ml\/docs\/bigqueryml-intro. [Online","author":"Google LLC.","year":"2019","unstructured":"Google LLC. 2019. Introduction to BigQuery ML. https:\/\/cloud.google.com\/bigquery-ml\/docs\/bigqueryml-intro. [Online; accessed 10-October-2019]."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2213836.2213875"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299869.3300066"},{"key":"e_1_3_2_1_33_1","volume-title":"OpenML Supervised Classification on adult. https:\/\/www.openml.org\/t\/7592. [Online","author":"ML.","year":"2020","unstructured":"OpenML. 2020. OpenML Supervised Classification on adult. https:\/\/www.openml.org\/t\/7592. [Online; accessed 1-December-2020]."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.14778\/3229863.3229867"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, EMNLP","author":"Socher Richard","year":"2013","unstructured":"Richard Socher, Alex Perelygin, Jean Wu, Jason Chuang, Christopher D. Manning, Andrew Y. Ng, and Christopher Potts. 2013. Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank. In Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, EMNLP 2013, 18--21 October 2013, Grand Hyatt Seattle, Seattle, Washington, USA, A meeting of SIGDAT, a Special Interest Group of the ACL. ACL, 1631--1642. https:\/\/aclanthology.org\/D13--1170\/"},{"key":"e_1_3_2_1_37_1","volume-title":"SQLFlow: Bridging Data and AI. https:\/\/sqlflow.org. [Online","year":"2020","unstructured":"SQLFlow. 2019. SQLFlow: Bridging Data and AI. https:\/\/sqlflow.org. [Online; accessed 1-December-2020]."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346320"},{"key":"e_1_3_2_1_39_1","volume-title":"XLA: Optimizing Compiler for Machine Learning. https:\/\/www.tensorflow.org\/xla. [Online","year":"2020","unstructured":"Tensorflow. 2020. XLA: Optimizing Compiler for Machine Learning. https:\/\/www.tensorflow.org\/xla. [Online; accessed 1-December-2020]."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419111.3421292"},{"key":"e_1_3_2_1_41_1","volume-title":"Asymptotic statistics","author":"Van der Vaart Aad W","unstructured":"Aad W Van der Vaart. 2000. Asymptotic statistics. Vol. 3. Cambridge university press."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2750549"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.14778\/2536354.2536356"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3389696"},{"key":"e_1_3_2_1_45_1","volume-title":"Proceedings of the 37th International Conference on Machine Learning (Proceedings of Machine Learning Research","author":"Wu Yinjun","year":"2020","unstructured":"Yinjun Wu, Edgar Dobriban, and Susan Davidson. 2020 a. DeltaGrad: Rapid retraining of machine learning models. In Proceedings of the 37th International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 119), Hal Daum\u00e9 III and Aarti Singh (Eds.). PMLR, 10355--10366. https:\/\/proceedings.mlr.press\/v119\/wu20b.html"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3380571"},{"key":"e_1_3_2_1_47_1","volume-title":"Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms. CoRR","author":"Xiao Han","year":"2017","unstructured":"Han Xiao, Kashif Rasul, and Roland Vollgraf. 2017. Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms. CoRR, Vol. abs\/1708.07747 (2017). arxiv: 1708.07747 http:\/\/arxiv.org\/abs\/1708.07747"},{"key":"e_1_3_2_1_48_1","volume-title":"Alexander Schwing, Murali Annavaram, and Salman Avestimehr.","author":"Yu Mingchao","year":"2018","unstructured":"Mingchao Yu, Zhifeng Lin, Krishna Narra, Songze Li, Youjie Li, Nam Sung Kim, Alexander Schwing, Murali Annavaram, and Salman Avestimehr. 2018. GradiVeQ: Vector Quantization for Bandwidth-Efficient Gradient Aggregation in Distributed CNN Training. In Advances in Neural Information Processing Systems, S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, and R. Garnett (Eds.), Vol. 31. Curran Associates, Inc., Redhook, NY, USA. https:\/\/proceedings.neurips.cc\/paper\/2018\/file\/cf05968255451bdefe3c5bc64d550517-Paper.pdf"},{"key":"e_1_3_2_1_49_1","volume-title":"Wide Residual Networks. CoRR","author":"Zagoruyko Sergey","year":"2016","unstructured":"Sergey Zagoruyko and Nikos Komodakis. 2016. Wide Residual Networks. CoRR, Vol. abs\/1605.07146 (2016). arxiv: 1605.07146 http:\/\/arxiv.org\/abs\/1605.07146"},{"key":"e_1_3_2_1_50_1","volume-title":"Proceedings of the Thirty-Second AAAI Conference on Artificial Intelligence, (AAAI-18)","author":"Zhang Xuezhou","year":"2018","unstructured":"Xuezhou Zhang, Xiaojin Zhu, and Stephen J. Wright. 2018. Training Set Debugging Using Trusted Items. In Proceedings of the Thirty-Second AAAI Conference on Artificial Intelligence, (AAAI-18), the 30th innovative Applications of Artificial Intelligence (IAAI-18), and the 8th AAAI Symposium on Educational Advances in Artificial Intelligence (EAAI-18), New Orleans, Louisiana, USA, February 2--7, 2018, Sheila A. McIlraith and Kilian Q. Weinberger (Eds.). AAAI Press, 4482--4489. https:\/\/www.aaai.org\/ocs\/index.php\/AAAI\/AAAI18\/paper\/view\/16155"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2020.3004555"}],"event":{"name":"SIGMOD\/PODS '22: International Conference on Management of Data","location":"Philadelphia PA USA","acronym":"SIGMOD\/PODS '22","sponsor":["SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the 2022 International Conference on Management of Data"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3514221.3517849","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3514221.3517849","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3514221.3517849","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:35Z","timestamp":1750188635000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3514221.3517849"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,10]]},"references-count":51,"alternative-id":["10.1145\/3514221.3517849","10.1145\/3514221"],"URL":"https:\/\/doi.org\/10.1145\/3514221.3517849","relation":{},"subject":[],"published":{"date-parts":[[2022,6,10]]},"assertion":[{"value":"2022-06-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}