{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T05:48:51Z","timestamp":1777873731699,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":79,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,8,3]]},"DOI":"10.1145\/3711896.3737422","type":"proceedings-article","created":{"date-parts":[[2025,8,3]],"date-time":"2025-08-03T21:04:26Z","timestamp":1754255066000},"page":"5482-5492","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["DCA-Bench: A Benchmark for Dataset Curation Agents"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0003-5449-2813","authenticated-orcid":false,"given":"Benhao","family":"Huang","sequence":"first","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3782-6888","authenticated-orcid":false,"given":"Yingzhuo","family":"Yu","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Champaign, IL, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1780-2513","authenticated-orcid":false,"given":"Jin","family":"Huang","sequence":"additional","affiliation":[{"name":"University of Michigan - Ann Arbor, Ann Arbor, MI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-0716-3449","authenticated-orcid":false,"given":"Xingjian","family":"Zhang","sequence":"additional","affiliation":[{"name":"University of Michigan - Ann Arbor, Ann Arbor, MI, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8292-5901","authenticated-orcid":false,"given":"Jiaqi W.","family":"Ma","sequence":"additional","affiliation":[{"name":"University of Illinois Urbana-Champaign, Champaign, IL, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,8,3]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Mart\u00edn Abadi Ashish Agarwal Paul Barham Eugene Brevdo Zhifeng Chen Craig Citro Greg S. Corrado Andy Davis Jeffrey Dean Matthieu Devin Sanjay Ghemawat Ian Goodfellow Andrew Harp Geoffrey Irving Michael Isard Yangqing Jia Rafal Jozefowicz Lukasz Kaiser Manjunath Kudlur Josh Levenberg Dandelion Man\u00e9 Rajat Monga Sherry Moore Derek Murray Chris Olah Mike Schuster Jonathon Shlens Benoit Steiner Ilya Sutskever Kunal Talwar Paul Tucker Vincent Vanhoucke Vijay Vasudevan Fernanda Vi\u00e9gas Oriol Vinyals Pete Warden Martin Wattenberg Martin Wicke Yuan Yu and Xiaoqiang Zheng. 2015. TensorFlow: Large-Scale Machine Learning on Heterogeneous Systems. https:\/\/www.tensorflow.org\/ Software available from tensorflow.org."},{"key":"e_1_3_2_2_2_1","unstructured":"Meta AI. 2024. Llama-3.3-70B-Instruct Model Card. https:\/\/huggingface.co\/meta-llama\/Llama-3.3-70B-Instruct"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.14569\/IJACSA.2022.01305109"},{"key":"e_1_3_2_2_4_1","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le and Charles Sutton. 2021. Program Synthesis with Large Language Models. arxiv: 2108.07732 [cs.PL]"},{"key":"e_1_3_2_2_5_1","unstructured":"AWS-Labs. 2024. Open Data Registry. https:\/\/github.com\/awslabs\/open-data-registry Accessed: 2024-11-18."},{"key":"e_1_3_2_2_6_1","unstructured":"Harrison Chase. 2022. LangChain. https:\/\/github.com\/langchain-ai\/langchain If you use this software please cite it as below.."},{"key":"e_1_3_2_2_7_1","unstructured":"Bei Chen Fengji Zhang Anh Nguyen Daoguang Zan Zeqi Lin Jian-Guang Lou and Weizhu Chen. 2022. CodeT: Code Generation with Generated Tests. arxiv: 2207.10397 [cs.CL]"},{"key":"e_1_3_2_2_8_1","unstructured":"Daoyuan Chen Yilun Huang Zhijian Ma Hesen Chen Xuchen Pan Ce Ge Dawei Gao Yuexiang Xie Zhaoyang Liu Jinyang Gao Yaliang Li Bolin Ding and Jingren Zhou. 2023b. Data-Juicer: A One-Stop Data Processing System for Large Language Models. arxiv: 2309.02033 [cs.LG] https:\/\/arxiv.org\/abs\/2309.02033"},{"key":"e_1_3_2_2_9_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. arxiv: 2107.03374 [cs.LG]"},{"key":"e_1_3_2_2_10_1","unstructured":"Yixiong Chen Li Liu and Chris Ding. 2023c. X-IQE: eXplainable Image Quality Evaluation for Text-to-Image Generation with Visual Large Language Models. arxiv: 2305.10843 [cs.CV]"},{"key":"e_1_3_2_2_11_1","volume-title":"Supersonic: Learning to Generate Source Code Optimizations in C\/C. arxiv: 2309.14846 [cs.SE]","author":"Chen Zimin","year":"2023","unstructured":"Zimin Chen, Sen Fang, and Martin Monperrus. 2023a. Supersonic: Learning to Generate Source Code Optimizations in C\/C. arxiv: 2309.14846 [cs.SE]"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.870"},{"key":"e_1_3_2_2_13_1","unstructured":"Cheng-Han Chiang and Hung yi Lee. 2023. A Closer Look into Automatic Evaluation Using Large Language Models. arxiv: 2310.05657 [cs.CL]"},{"key":"e_1_3_2_2_14_1","unstructured":"DeepSeek-AI Daya Guo Dejian Yang Haowei Zhang Junxiao Song Ruoyu Zhang Runxin Xu Qihao Zhu Shirong Ma Peiyi Wang Xiao Bi Xiaokang Zhang Xingkai Yu Yu Wu Z. F. Wu Zhibin Gou Zhihong Shao Zhuoshu Li Ziyi Gao Aixin Liu Bing Xue Bingxuan Wang Bochao Wu Bei Feng Chengda Lu Chenggang Zhao Chengqi Deng Chenyu Zhang Chong Ruan Damai Dai Deli Chen Dongjie Ji Erhang Li Fangyun Lin Fucong Dai Fuli Luo Guangbo Hao Guanting Chen Guowei Li H. Zhang Han Bao Hanwei Xu Haocheng Wang Honghui Ding Huajian Xin Huazuo Gao Hui Qu Hui Li Jianzhong Guo Jiashi Li Jiawei Wang Jingchang Chen Jingyang Yuan Junjie Qiu Junlong Li J. L. Cai Jiaqi Ni Jian Liang Jin Chen Kai Dong Kai Hu Kaige Gao Kang Guan Kexin Huang Kuai Yu Lean Wang Lecong Zhang Liang Zhao Litong Wang Liyue Zhang Lei Xu Leyi Xia Mingchuan Zhang Minghua Zhang Minghui Tang Meng Li Miaojun Wang Mingming Li Ning Tian Panpan Huang Peng Zhang Qiancheng Wang Qinyu Chen Qiushi Du Ruiqi Ge Ruisong Zhang Ruizhe Pan Runji Wang R. J. Chen R. L. Jin Ruyi Chen Shanghao Lu Shangyan Zhou Shanhuang Chen Shengfeng Ye Shiyu Wang Shuiping Yu Shunfeng Zhou Shuting Pan S. S. Li Shuang Zhou Shaoqing Wu Shengfeng Ye Tao Yun Tian Pei Tianyu Sun T. Wang Wangding Zeng Wanjia Zhao Wen Liu Wenfeng Liang Wenjun Gao Wenqin Yu Wentao Zhang W. L. Xiao Wei An Xiaodong Liu Xiaohan Wang Xiaokang Chen Xiaotao Nie Xin Cheng Xin Liu Xin Xie Xingchao Liu Xinyu Yang Xinyuan Li Xuecheng Su Xuheng Lin X. Q. Li Xiangyue Jin Xiaojin Shen Xiaosha Chen Xiaowen Sun Xiaoxiang Wang Xinnan Song Xinyi Zhou Xianzu Wang Xinxia Shan Y. K. Li Y. Q. Wang Y. X. Wei Yang Zhang Yanhong Xu Yao Li Yao Zhao Yaofeng Sun Yaohui Wang Yi Yu Yichao Zhang Yifan Shi Yiliang Xiong Ying He Yishi Piao Yisong Wang Yixuan Tan Yiyang Ma Yiyuan Liu Yongqiang Guo Yuan Ou Yuduan Wang Yue Gong Yuheng Zou Yujia He Yunfan Xiong Yuxiang Luo Yuxiang You Yuxuan Liu Yuyang Zhou Y. X. Zhu Yanhong Xu Yanping Huang Yaohui Li Yi Zheng Yuchen Zhu Yunxian Ma Ying Tang Yukun Zha Yuting Yan Z. Z. Ren Zehui Ren Zhangli Sha Zhe Fu Zhean Xu Zhenda Xie Zhengyan Zhang Zhewen Hao Zhicheng Ma Zhigang Yan Zhiyu Wu Zihui Gu Zijia Zhu Zijun Liu Zilin Li Ziwei Xie Ziyang Song Zizheng Pan Zhen Huang Zhipeng Xu Zhongyu Zhang and Zhen Zhang. 2025 a. DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning. arxiv: 2501.12948 [cs.CL] https:\/\/arxiv.org\/abs\/2501.12948"},{"key":"e_1_3_2_2_15_1","unstructured":"DeepSeek-AI Aixin Liu Bei Feng Bing Xue Bingxuan Wang Bochao Wu Chengda Lu Chenggang Zhao Chengqi Deng Chenyu Zhang Chong Ruan Damai Dai Daya Guo Dejian Yang Deli Chen Dongjie Ji Erhang Li Fangyun Lin Fucong Dai Fuli Luo Guangbo Hao Guanting Chen Guowei Li H. Zhang Han Bao Hanwei Xu Haocheng Wang Haowei Zhang Honghui Ding Huajian Xin Huazuo Gao Hui Li Hui Qu J. L. Cai Jian Liang Jianzhong Guo Jiaqi Ni Jiashi Li Jiawei Wang Jin Chen Jingchang Chen Jingyang Yuan Junjie Qiu Junlong Li Junxiao Song Kai Dong Kai Hu Kaige Gao Kang Guan Kexin Huang Kuai Yu Lean Wang Lecong Zhang Lei Xu Leyi Xia Liang Zhao Litong Wang Liyue Zhang Meng Li Miaojun Wang Mingchuan Zhang Minghua Zhang Minghui Tang Mingming Li Ning Tian Panpan Huang Peiyi Wang Peng Zhang Qiancheng Wang Qihao Zhu Qinyu Chen Qiushi Du R. J. Chen R. L. Jin Ruiqi Ge Ruisong Zhang Ruizhe Pan Runji Wang Runxin Xu Ruoyu Zhang Ruyi Chen S. S. Li Shanghao Lu Shangyan Zhou Shanhuang Chen Shaoqing Wu Shengfeng Ye Shengfeng Ye Shirong Ma Shiyu Wang Shuang Zhou Shuiping Yu Shunfeng Zhou Shuting Pan T. Wang Tao Yun Tian Pei Tianyu Sun W. L. Xiao Wangding Zeng Wanjia Zhao Wei An Wen Liu Wenfeng Liang Wenjun Gao Wenqin Yu Wentao Zhang X. Q. Li Xiangyue Jin Xianzu Wang Xiao Bi Xiaodong Liu Xiaohan Wang Xiaojin Shen Xiaokang Chen Xiaokang Zhang Xiaosha Chen Xiaotao Nie Xiaowen Sun Xiaoxiang Wang Xin Cheng Xin Liu Xin Xie Xingchao Liu Xingkai Yu Xinnan Song Xinxia Shan Xinyi Zhou Xinyu Yang Xinyuan Li Xuecheng Su Xuheng Lin Y. K. Li Y. Q. Wang Y. X. Wei Y. X. Zhu Yang Zhang Yanhong Xu Yanhong Xu Yanping Huang Yao Li Yao Zhao Yaofeng Sun Yaohui Li Yaohui Wang Yi Yu Yi Zheng Yichao Zhang Yifan Shi Yiliang Xiong Ying He Ying Tang Yishi Piao Yisong Wang Yixuan Tan Yiyang Ma Yiyuan Liu Yongqiang Guo Yu Wu Yuan Ou Yuchen Zhu Yuduan Wang Yue Gong Yuheng Zou Yujia He Yukun Zha Yunfan Xiong Yunxian Ma Yuting Yan Yuxiang Luo Yuxiang You Yuxuan Liu Yuyang Zhou Z. F. Wu Z. Z. Ren Zehui Ren Zhangli Sha Zhe Fu Zhean Xu Zhen Huang Zhen Zhang Zhenda Xie Zhengyan Zhang Zhewen Hao Zhibin Gou Zhicheng Ma Zhigang Yan Zhihong Shao Zhipeng Xu Zhiyu Wu Zhongyu Zhang Zhuoshu Li Zihui Gu Zijia Zhu Zijun Liu Zilin Li Ziwei Xie Ziyang Song Ziyi Gao and Zizheng Pan. 2025 b. DeepSeek-V3 Technical Report. arxiv: 2412.19437 [cs.CL] https:\/\/arxiv.org\/abs\/2412.19437"},{"key":"e_1_3_2_2_16_1","volume-title":"Hantian Ding, Ming Tan, Nihal Jain, Murali Krishna Ramanathan, Ramesh Nallapati, Parminder Bhatia, Dan Roth, and Bing Xiang.","author":"Ding Yangruibo","year":"2023","unstructured":"Yangruibo Ding, Zijian Wang, Wasi Uddin Ahmad, Hantian Ding, Ming Tan, Nihal Jain, Murali Krishna Ramanathan, Ramesh Nallapati, Parminder Bhatia, Dan Roth, and Bing Xiang. 2023. CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion. arxiv: 2310.11248 [cs.LG]"},{"key":"e_1_3_2_2_17_1","unstructured":"Abhimanyu Dubey Abhinav Jauhri Abhinav Pandey Abhishek Kadian Ahmad Al-Dahle Aiesha Letman Akhil Mathur Alan Schelten Amy Yang Angela Fan et al. 2024. The llama 3 herd of models. arXiv preprint arXiv:2407.21783 (2024)."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-FoSE59343.2023.00008"},{"key":"e_1_3_2_2_19_1","unstructured":"Sidong Feng and Chunyang Chen. 2023. Prompting Is All You Need: Automated Android Bug Replay with Large Language Models. arxiv: 2306.01987 [cs.SE]"},{"key":"e_1_3_2_2_20_1","unstructured":"FiveThirtyEight. 2024. Data and code behind the articles and graphics at FiveThirtyEight. https:\/\/github.com\/fivethirtyeight\/data Accessed: 2024-11-18."},{"key":"e_1_3_2_2_21_1","unstructured":"Ruyi Gan Ziwei Wu Renliang Sun Junyu Lu Xiaojun Wu Dixiang Zhang Kunhao Pan Junqing He Yuanhe Tian Ping Yang Qi Yang Hao Wang Jiaxing Zhang and Yan Song. 2024. Ziya2: Data-centric Learning is All LLMs Need. arxiv: 2311.03301 [cs.CL]"},{"key":"e_1_3_2_2_22_1","volume-title":"Roshanak Zilouchian Moghaddam, and Neel Sundaresan","author":"Garg Spandan","year":"2024","unstructured":"Spandan Garg, Roshanak Zilouchian Moghaddam, and Neel Sundaresan. 2024. RAPGen: An Approach for Fixing Code Inefficiencies in Zero-Shot. arxiv: 2306.17077 [cs.SE]"},{"key":"e_1_3_2_2_23_1","volume-title":"Hanna Wallach, Hal Daum\u00e9 III au2, and Kate Crawford.","author":"Gebru Timnit","year":"2021","unstructured":"Timnit Gebru, Jamie Morgenstern, Briana Vecchione, Jennifer Wortman Vaughan, Hanna Wallach, Hal Daum\u00e9 III au2, and Kate Crawford. 2021. Datasheets for Datasets. arxiv: 1803.09010 [cs.DB]"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2023.107268"},{"key":"e_1_3_2_2_25_1","unstructured":"Siyuan Guo Cheng Deng Ying Wen Hechang Chen Yi Chang and Jun Wang. 2024. DS-Agent: Automated Data Science by Empowering Large Language Models with Case-Based Reasoning. arxiv: 2402.17453 [cs.LG] https:\/\/arxiv.org\/abs\/2402.17453"},{"key":"e_1_3_2_2_26_1","volume-title":"Shanmukha Guttula, Abhinav Jain, Lokesh Nagalapatti, Sameep Mehta, Sandeep Hans, Pranay Lohia, Aniya Aggarwal, and Diptikalyan Saha.","author":"Gupta Nitin","year":"2021","unstructured":"Nitin Gupta, Hima Patel, Shazia Afzal, Naveen Panwar, Ruhi Sharma Mittal, Shanmukha Guttula, Abhinav Jain, Lokesh Nagalapatti, Sameep Mehta, Sandeep Hans, Pranay Lohia, Aniya Aggarwal, and Diptikalyan Saha. 2021. Data Quality Toolkit: Automatic assessment of data quality and remediation for machine learning datasets. arxiv: 2108.05935 [cs.LG]"},{"key":"e_1_3_2_2_27_1","volume-title":"Smith","author":"Gururangan Suchin","year":"2018","unstructured":"Suchin Gururangan, Swabha Swayamdipta, Omer Levy, Roy Schwartz, Samuel R. Bowman, and Noah A. Smith. 2018. Annotation Artifacts in Natural Language Inference Data. arxiv: 1803.02324 [cs.CL]"},{"key":"e_1_3_2_2_28_1","unstructured":"Shibo Hao Yi Gu Haotian Luo Tianyang Liu Xiyan Shao Xinyuan Wang Shuhua Xie Haodi Ma Adithya Samavedhi Qiyue Gao Zhen Wang and Zhiting Hu. 2024. LLM Reasoners: New Evaluation Library and Analysis of Step-by-Step Reasoning with Large Language Models. arxiv: 2404.05221 [cs.CL]"},{"key":"e_1_3_2_2_29_1","unstructured":"Hangfeng He Hongming Zhang and Dan Roth. 2024b. SocREval: Large Language Models with the Socratic Method for Reference-Free Reasoning Evaluation. arxiv: 2310.00074 [cs.CL]"},{"key":"e_1_3_2_2_30_1","unstructured":"Qianyu He Jie Zeng Wenhao Huang Lina Chen Jin Xiao Qianxi He Xunzhe Zhou Lida Chen Xintao Wang Yuncheng Huang Haoning Ye Zihan Li Shisong Chen Yikai Zhang Zhouhong Gu Jiaqing Liang and Yanghua Xiao. 2024a. Can Large Language Models Understand Real-World Complex Instructions? arxiv: 2309.09150 [cs.CL]"},{"key":"e_1_3_2_2_31_1","volume-title":"Data Interpreter: An LLM Agent For Data Science. arxiv: 2402.18679 [cs.AI] https:\/\/arxiv.org\/abs\/2402.18679","author":"Hong Sirui","year":"2024","unstructured":"Sirui Hong, Yizhang Lin, Bang Liu, Bangbang Liu, Binhao Wu, Ceyao Zhang, Chenxing Wei, Danyang Li, Jiaqi Chen, Jiayi Zhang, Jinlin Wang, Li Zhang, Lingyao Zhang, Min Yang, Mingchen Zhuge, Taicheng Guo, Tuo Zhou, Wei Tao, Xiangru Tang, Xiangtao Lu, Xiawu Zheng, Xinbing Liang, Yaying Fei, Yuheng Cheng, Zhibin Gou, Zongze Xu, and Chenglin Wu. 2024. Data Interpreter: An LLM Agent For Data Science. arxiv: 2402.18679 [cs.AI] https:\/\/arxiv.org\/abs\/2402.18679"},{"key":"e_1_3_2_2_32_1","volume-title":"Zijuan Lin, Liyang Zhou, Chenyu Ran, Lingfeng Xiao, Chenglin Wu, and J\u00fcrgen Schmidhuber.","author":"Hong Sirui","year":"2023","unstructured":"Sirui Hong, Mingchen Zhuge, Jonathan Chen, Xiawu Zheng, Yuheng Cheng, Ceyao Zhang, Jinlin Wang, Zili Wang, Steven Ka Shing Yau, Zijuan Lin, Liyang Zhou, Chenyu Ran, Lingfeng Xiao, Chenglin Wu, and J\u00fcrgen Schmidhuber. 2023. MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework. arxiv: 2308.00352 [cs.AI]"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3406477"},{"key":"e_1_3_2_2_34_1","volume-title":"The nature of problem finding in students' scientific inquiry","author":"Jay Eileen S","unstructured":"Eileen S Jay. 1996. The nature of problem finding in students' scientific inquiry. Harvard University."},{"key":"e_1_3_2_2_35_1","unstructured":"Carlos E. Jimenez John Yang Alexander Wettig Shunyu Yao Kexin Pei Ofir Press and Karthik Narasimhan. 2024. SWE-bench: Can Language Models Resolve Real-World GitHub Issues? arxiv: 2310.06770 [cs.CL]"},{"key":"e_1_3_2_2_36_1","unstructured":"Kaggle. [n. d.]. Creating a Dataset. https:\/\/www.kaggle.com\/docs\/datasets#creating-a-dataset. Accessed: 2024-09-26."},{"key":"e_1_3_2_2_37_1","volume-title":"Scaling Laws for Neural Language Models. arxiv","author":"Kaplan Jared","year":"2001","unstructured":"Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B. Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. 2020. Scaling Laws for Neural Language Models. arxiv: 2001.08361 [cs.LG]"},{"key":"e_1_3_2_2_38_1","unstructured":"Hannah Rose Kirk Abeba Birhane Bertie Vidgen and Leon Derczynski. 2023. Handling and Presenting Harmful Text in NLP Research. arxiv: 2204.14256 [cs.CL]"},{"key":"e_1_3_2_2_39_1","volume-title":"Richard Eckart de Castilho, and Iryna Gurevych","author":"Klie Jan-Christoph","year":"2024","unstructured":"Jan-Christoph Klie, Richard Eckart de Castilho, and Iryna Gurevych. 2024. Analyzing Dataset Annotation Quality Management in the Wild. arxiv: 2307.08153 [cs.CL]"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1162\/coli_a_00464"},{"key":"e_1_3_2_2_41_1","unstructured":"Aobo Kong Shiwan Zhao Hao Chen Qicheng Li Yong Qin Ruiqi Sun Xin Zhou Enzhi Wang and Xiaohang Dong. 2024. Better Zero-Shot Reasoning with Role-Play Prompting. arxiv: 2308.07702 [cs.CL]"},{"key":"e_1_3_2_2_42_1","volume-title":"Daniel Fried, Sida Wang, and Tao Yu.","author":"Lai Yuhang","year":"2022","unstructured":"Yuhang Lai, Chengxi Li, Yiming Wang, Tianyi Zhang, Ruiqi Zhong, Luke Zettlemoyer, Scott Wen tau Yih, Daniel Fried, Sida Wang, and Tao Yu. 2022. DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation. arxiv: 2211.11501 [cs.SE]"},{"key":"e_1_3_2_2_43_1","volume-title":"Tim Rockt\u00e4schel, Sebastian Riedel, and Douwe Kiela.","author":"Lewis Patrick","year":"2021","unstructured":"Patrick Lewis, Ethan Perez, Aleksandra Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\u00fcttler, Mike Lewis, Wen tau Yih, Tim Rockt\u00e4schel, Sebastian Riedel, and Douwe Kiela. 2021. Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks. arxiv: 2005.11401 [cs.CL]"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.4817768"},{"key":"e_1_3_2_2_45_1","unstructured":"Tianyang Liu Canwen Xu and Julian McAuley. 2023b. RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems. arxiv: 2306.03091 [cs.CL]"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"crossref","unstructured":"Yang Liu Dan Iter Yichong Xu Shuohang Wang Ruochen Xu and Chenguang Zhu. 2023a. G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment. arxiv: 2303.16634 [cs.CL]","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"e_1_3_2_2_47_1","unstructured":"Shayne Longpre Robert Mahari Anthony Chen Naana Obeng-Marnu Damien Sileo William Brannon Niklas Muennighoff Nathan Khazam Jad Kabbara Kartik Perisetla Xinyi Wu Enrico Shippole Kurt Bollacker Tongshuang Wu Luis Villa Sandy Pentland and Sara Hooker. 2023. The Data Provenance Initiative: A Large Scale Audit of Dataset Licensing & Attribution in AI. arxiv: 2310.16787 [cs.CL] https:\/\/arxiv.org\/abs\/2310.16787"},{"key":"e_1_3_2_2_48_1","unstructured":"Ziyang Luo Can Xu Pu Zhao Qingfeng Sun Xiubo Geng Wenxiang Hu Chongyang Tao Jing Ma Qingwei Lin and Daxin Jiang. 2023. WizardCoder: Empowering Code Large Language Models with Evol-Instruct. arxiv: 2306.08568 [cs.CL]"},{"key":"e_1_3_2_2_49_1","volume-title":"Yiwen Tu, Chenshu Zhu, and Qiaozhu Mei.","author":"Ma Jiaqi","year":"2022","unstructured":"Jiaqi Ma, Xingjian Zhang, Hezheng Fan, Jin Huang, Tianyue Li, Ting Wei Li, Yiwen Tu, Chenshu Zhu, and Qiaozhu Mei. 2022. Graph Learning Indexer: A Contributor-Friendly and Metadata-Rich Platform for Graph Learning Benchmarks. arxiv: 2212.04537 [cs.LG]"},{"key":"e_1_3_2_2_50_1","volume-title":"Fair Enough: Searching for Sufficient Measures of Fairness. arXiv","author":"Majumder S.","year":"2022","unstructured":"S. Majumder, J. Chakraborty, G. R. Bai, K. T. Stolee, and T. Menzies. 2022. Fair Enough: Searching for Sufficient Measures of Fairness. arXiv (2022). http:\/\/arxiv.org\/abs\/2110.13029 Accessed: 2024-08-13."},{"key":"e_1_3_2_2_51_1","unstructured":"Ze Mao Yang Xu and Erick Suarez. 2023. Dataset Management Platform for Machine Learning. arxiv: 2303.08301 [cs.DB]"},{"key":"e_1_3_2_2_52_1","unstructured":"Unit Mesh. 2024. auto-dev. https:\/\/github.com\/unit-mesh\/auto-dev."},{"key":"e_1_3_2_2_53_1","unstructured":"OpenAI. 2023. GPT-3.5-Turbo. https:\/\/platform.openai.com\/docs\/models\/gpt-3-5"},{"key":"e_1_3_2_2_54_1","unstructured":"OpenAI. 2024a. GPT-4 Technical Report. arxiv: 2303.08774 [cs.CL]"},{"key":"e_1_3_2_2_55_1","unstructured":"OpenAI. 2024b. O3 Mini System Card. https:\/\/openai.com\/index\/o3-mini-system-card\/ Accessed: 2025-02-22."},{"key":"e_1_3_2_2_56_1","unstructured":"OpenAI. 2024c. Six Strategies for Getting Better Results. https:\/\/platform.openai.com\/docs\/guides\/prompt-engineering\/six-strategies-for-getting-better-results. Accessed: 2024-05-28."},{"key":"e_1_3_2_2_57_1","unstructured":"OpenAI Aaron Hurst Adam Lerer Adam P. Goucher and et al. 2024. GPT-4o System Card. arxiv: 2410.21276 [cs.CL] https:\/\/arxiv.org\/abs\/2410.21276"},{"key":"e_1_3_2_2_58_1","unstructured":"Russell A Poldrack Thomas Lu and Ga\u0161per Begu\u0161. 2023. AI-assisted coding: Experiments with GPT-4. arxiv: 2304.13187 [cs.AI]"},{"key":"e_1_3_2_2_59_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Romain Sauvestre Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2024. Code Llama: Open Foundation Models for Code. arxiv: 2308.12950 [cs.CL]"},{"key":"e_1_3_2_2_60_1","volume-title":"Reflexion: Language Agents with Verbal Reinforcement Learning. arxiv: 2303.11366 [cs.AI]","author":"Shinn Noah","year":"2023","unstructured":"Noah Shinn, Federico Cassano, Edward Berman, Ashwin Gopinath, Karthik Narasimhan, and Shunyu Yao. 2023. Reflexion: Language Agents with Verbal Reinforcement Learning. arxiv: 2303.11366 [cs.AI]"},{"key":"e_1_3_2_2_61_1","volume-title":"Abubakar Abid, Adam Fisch, Adam R. Brown, and et al. Adam Santoro.","author":"Srivastava Aarohi","year":"2023","unstructured":"Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R. Brown, and et al. Adam Santoro. 2023. Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models. arxiv: 2206.04615 [cs.CL]"},{"key":"e_1_3_2_2_62_1","volume-title":"MAGIS: LLM-Based Multi-Agent Framework for GitHub Issue Resolution. arxiv: 2403.17927 [cs.SE]","author":"Tao Wei","year":"2024","unstructured":"Wei Tao, Yucheng Zhou, Wenqiang Zhang, and Yu Cheng. 2024. MAGIS: LLM-Based Multi-Agent Framework for GitHub Issue Resolution. arxiv: 2403.17927 [cs.SE]"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2641190.2641198"},{"key":"e_1_3_2_2_64_1","unstructured":"Pat Verga Sebastian Hofstatter Sophia Althammer Yixuan Su Aleksandra Piktus Arkady Arkhangorodsky Minjie Xu Naomi White and Patrick Lewis. 2024. Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models. arxiv: 2404.18796 [cs.CL]"},{"key":"e_1_3_2_2_65_1","volume-title":"Detecting label errors in token classification data. arXiv preprint arXiv:2210.03920","author":"Wang Wei-Chen","year":"2022","unstructured":"Wei-Chen Wang and Jonas Mueller. 2022. Detecting label errors in token classification data. arXiv preprint arXiv:2210.03920 (2022)."},{"key":"e_1_3_2_2_66_1","unstructured":"Zige Wang Wanjun Zhong Yufei Wang Qi Zhu Fei Mi Baojun Wang Lifeng Shang Xin Jiang and Qun Liu. 2023. Data Management For Large Language Models: A Survey. arxiv: 2312.01700 [cs.CL]"},{"key":"e_1_3_2_2_67_1","volume-title":"Donkii: Can Annotation Error Detection Methods Find Errors in Instruction-Tuning Datasets? arXiv preprint arXiv:2309.01669","author":"Weber-Genzel Leon","year":"2023","unstructured":"Leon Weber-Genzel, Robert Litschko, Ekaterina Artemova, and Barbara Plank. 2023. Donkii: Can Annotation Error Detection Methods Find Errors in Instruction-Tuning Datasets? arXiv preprint arXiv:2309.01669 (2023)."},{"key":"e_1_3_2_2_68_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2023","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed Chi, Quoc Le, and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arxiv: 2201.11903 [cs.CL]"},{"key":"e_1_3_2_2_69_1","volume-title":"The Free Encyclopedia. https:\/\/en.wikipedia.org\/w\/index.php?title=Problem_finding&oldid=1122334455 [Online","author":"Wikipedia","year":"2024","unstructured":"Wikipedia contributors. 2024. Problem finding - Wikipedia, The Free Encyclopedia. https:\/\/en.wikipedia.org\/w\/index.php?title=Problem_finding&oldid=1122334455 [Online; accessed 18-April-2024]."},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1038\/sdata.2016.18"},{"key":"e_1_3_2_2_71_1","volume-title":"Ryen W White, Doug Burger, and Chi Wang.","author":"Wu Qingyun","year":"2023","unstructured":"Qingyun Wu, Gagan Bansal, Jieyu Zhang, Yiran Wu, Beibin Li, Erkang Zhu, Li Jiang, Xiaoyun Zhang, Shaokun Zhang, Jiale Liu, Ahmed Hassan Awadallah, Ryen W White, Doug Burger, and Chi Wang. 2023a. AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation. arxiv: 2308.08155 [cs.AI] https:\/\/arxiv.org\/abs\/2308.08155"},{"key":"e_1_3_2_2_72_1","unstructured":"Yonghao Wu Zheng Li Jie M. Zhang Mike Papadakis Mark Harman and Yong Liu. 2023b. Large Language Models in Fault Localisation. arxiv: 2308.15276 [cs.SE]"},{"key":"e_1_3_2_2_73_1","unstructured":"Xinyu Yang Weixin Liang and James Zou. 2024. Navigating Dataset Documentations in AI: A Large-Scale Analysis of Dataset Cards on Hugging Face. arxiv: 2401.13822 [cs.LG]"},{"key":"e_1_3_2_2_74_1","unstructured":"Zhiyuan You Zheyuan Li Jinjin Gu Zhenfei Yin Tianfan Xue and Chao Dong. 2024. Depicting Beyond Scores: Advancing Image Quality Assessment through Multi-modal Language Models. arxiv: 2312.08962 [cs.CV]"},{"key":"e_1_3_2_2_75_1","volume-title":"Parsel: Algorithmic Reasoning with Language Models by Composing Decompositions. arxiv: 2212.10561 [cs.CL]","author":"Zelikman Eric","year":"2023","unstructured":"Eric Zelikman, Qian Huang, Gabriel Poesia, Noah D. Goodman, and Nick Haber. 2023. Parsel: Algorithmic Reasoning with Language Models by Composing Decompositions. arxiv: 2212.10561 [cs.CL]"},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623322"},{"key":"e_1_3_2_2_77_1","volume-title":"Shiyi Cao, Christos Kozyrakis, Ion Stoica, Joseph E. Gonzalez, Clark Barrett, and Ying Sheng.","author":"Zheng Lianmin","year":"2024","unstructured":"Lianmin Zheng, Liangsheng Yin, Zhiqiang Xie, Chuyue Sun, Jeff Huang, Cody Hao Yu, Shiyi Cao, Christos Kozyrakis, Ion Stoica, Joseph E. Gonzalez, Clark Barrett, and Ying Sheng. 2024. SGLang: Efficient Execution of Structured Language Model Programs. arxiv: 2312.07104 [cs.AI] https:\/\/arxiv.org\/abs\/2312.07104"},{"key":"e_1_3_2_2_78_1","volume-title":"Oasis: Data Curation and Assessment System for Pretraining of Large Language Models. arxiv: 2311.12537 [cs.CL]","author":"Zhou Tong","year":"2023","unstructured":"Tong Zhou, Yubo Chen, Pengfei Cao, Kang Liu, Jun Zhao, and Shengping Liu. 2023. Oasis: Data Curation and Assessment System for Pretraining of Large Language Models. arxiv: 2311.12537 [cs.CL]"},{"key":"e_1_3_2_2_79_1","unstructured":"Xuanhe Zhou Xinyang Zhao and Guoliang Li. 2024. LLM-Enhanced Data Management. arxiv: 2402.02643 [cs.DB] https:\/\/arxiv.org\/abs\/2402.02643"}],"event":{"name":"KDD '25: The 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Toronto ON Canada","acronym":"KDD '25","sponsor":["SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3711896.3737422","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T18:16:42Z","timestamp":1777573002000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3711896.3737422"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,8,3]]},"references-count":79,"alternative-id":["10.1145\/3711896.3737422","10.1145\/3711896"],"URL":"https:\/\/doi.org\/10.1145\/3711896.3737422","relation":{},"subject":[],"published":{"date-parts":[[2025,8,3]]},"assertion":[{"value":"2025-08-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}