{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,26]],"date-time":"2026-10-26T16:39:54Z","timestamp":1793032794212,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":71,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,8,8]],"date-time":"2026-08-08T00:00:00Z","timestamp":1786147200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"Defense Advanced Research Projects Agency","award":["HR001126CE054"],"award-info":[{"award-number":["HR001126CE054"]}]},{"name":"NIH (National Institutes of Health)","award":["OT2OD038003"],"award-info":[{"award-number":["OT2OD038003"]}]},{"name":"NIH (National Institutes of Health)","award":["U54OD036472"],"award-info":[{"award-number":["U54OD036472"]}]},{"name":"NIH (National Institutes of Health)","award":["U54HG012517"],"award-info":[{"award-number":["U54HG012517"]}]},{"name":"NIH (National Institutes of Health)","award":["U54DK097771"],"award-info":[{"award-number":["U54DK097771"]}]},{"name":"NSF (National Science Foundation)","award":["2312501"],"award-info":[{"award-number":["2312501"]}]},{"name":"NSF (National Science Foundation)","award":["2106859"],"award-info":[{"award-number":["2106859"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,8,9]]},"DOI":"10.1145\/3770855.3818888","type":"proceedings-article","created":{"date-parts":[[2026,8,6]],"date-time":"2026-08-06T17:15:37Z","timestamp":1786036537000},"page":"12868-12879","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["MatSciBench: Benchmarking the Reasoning Ability of Large Language Models in Materials Science"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-9899-8695","authenticated-orcid":false,"given":"Junkai","family":"Zhang","sequence":"first","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-8557-2012","authenticated-orcid":false,"given":"Jingru","family":"Gan","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4766-2823","authenticated-orcid":false,"given":"Xiaoxuan","family":"Wang","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0181-9961","authenticated-orcid":false,"given":"Zian","family":"Jia","sequence":"additional","affiliation":[{"name":"Department of Materials Science and Engineering, University of Pennsylvania, Philadelphia, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-8132-9337","authenticated-orcid":false,"given":"Changquan","family":"Gu","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7599-9961","authenticated-orcid":false,"given":"Jianpeng","family":"Chen","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Virginia Tech, Blacksburg, Virginia, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2205-5304","authenticated-orcid":false,"given":"Yanqiao","family":"Zhu","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4011-0568","authenticated-orcid":false,"given":"Mingyu Derek","family":"Ma","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7065-2990","authenticated-orcid":false,"given":"Dawei","family":"Zhou","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Virginia Tech, Blacksburg, Virginia, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6741-9741","authenticated-orcid":false,"given":"Ling","family":"Li","sequence":"additional","affiliation":[{"name":"Department of Materials Science and Engineering, University of Pennsylvania, Philadelphia, Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8180-2886","authenticated-orcid":false,"given":"Wei","family":"Wang","sequence":"additional","affiliation":[{"name":"Computer Science Department, University of California, Los Angeles, Los Angeles, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,8,8]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Nuclearqa: A human-made benchmark for language models for the nuclear domain. arXiv preprint arXiv:2310.10920","author":"Acharya Anurag","year":"2023","unstructured":"Anurag Acharya, Sai Munikoti, Aaron Hellinger, Sara Smith, Sridevi Wagle, and Sameera Horawalavithana. 2023. Nuclearqa: A human-made benchmark for language models for the nuclear domain. arXiv preprint arXiv:2310.10920 (2023)."},{"key":"e_1_3_2_1_2_1","volume-title":"Mara Schilling-Wilhelmi, N M Anoop Krishnan, and Kevin Maik Jablonka.","author":"Alampara Nawaf","year":"2024","unstructured":"Nawaf Alampara, Indrajeet Mandal, Pranav Khetarpal, Hargun Singh Grover, Mara Schilling-Wilhelmi, N M Anoop Krishnan, and Kevin Maik Jablonka. 2024. MaCBench: A multimodal chemistry and materials science benchmark. In AI for Accelerated Materials Design - NeurIPS 2024. https:\/\/openreview.net\/forum?id=Q2PNocDcp6"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-97-2262-4_5"},{"key":"e_1_3_2_1_4_1","unstructured":"Anthropic. 2025. Claude 3.7 Sonnet and Claude Code. https:\/\/www.anthropic.com\/news\/claude-3-7-sonnet. Accessed: 2025-05-15."},{"key":"e_1_3_2_1_5_1","unstructured":"Anthropic. 2025. Introducing Claude 4. https:\/\/www.anthropic.com\/news\/claude-4."},{"key":"e_1_3_2_1_6_1","volume-title":"Materials: Engineering, Science, Processing and Design (4 ed.). Butterworth-Heinemann","author":"Ashby Michael F.","year":"2019","unstructured":"Michael F. Ashby, Hugh Shercliff, and David Cebon. 2019. Materials: Engineering, Science, Processing and Design (4 ed.). Butterworth-Heinemann, Oxford, UK."},{"key":"e_1_3_2_1_7_1","volume-title":"Wright","author":"Askeland Donald R.","year":"2011","unstructured":"Donald R. Askeland, Pradeep P. Fulay, and Wendelin J. Wright. 2011. The Science and Engineering of Materials, SI Edition (6th ed.). CL-Engineering."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"M. W. Barsoum. 2003. Fundamentals of ceramics.","DOI":"10.1887\/0750309024"},{"key":"e_1_3_2_1_9_1","volume-title":"Yingyan Celine Lin, and Pavlo Molchanov","author":"Belcak Peter","year":"2025","unstructured":"Peter Belcak, Greg Heinrich, Shizhe Diao, Yonggan Fu, Xin Dong, Saurav Muralidharan, Yingyan Celine Lin, and Pavlo Molchanov. 2025. Small Language Models are the Future of Agentic AI. arXiv preprint arXiv:2506.02153 (2025)."},{"key":"e_1_3_2_1_10_1","unstructured":"ByteDance Seed Jiaze Chen Tiantian Fan Xin Liu Lingjun Liu Zhiqi Lin Mingxuan Wang Chengyi Wang Xiangpeng Wei Wenyuan Xu Yufeng Yuan Yu Yue Lin Yan Qiying Yu Xiaochen Zuo Chi Zhang Ruofei Zhu Zhecheng An Zhihao Bai Yu Bao Xingyan Bin Jiangjie Chen Feng Chen Hongmin Chen Riwei Chen Liangqiang Chen Zixin Chen Jinsong Chen Siyan Chen Kaiyuan Chen Zhi Chen Jin Chen Jiecao Chen Jinxin Chi Weinan Dai Ning Dai Jiahui Dai Shihan Dou Yantao Du Zhengyin Du Jianhui Duan Chen Dun Ting-Han Fan Jiazhan Feng Junda Feng Ziyuan Feng Yuwei Fu Wenqi Fu Hanjie Fu Hao Ge Hongyi Guo Mingji Han Li Han Wenhao Hao Xintong Hao Qianyu He Jerry He Feng He Wen Heng Zehua Hong Qi Hou Liang Hu Shengding Hu Nan Hu Kai Hua Qi Huang Ziyue Huang Hongzhi Huang Zihao Huang Ting Huang Wenhao Huang Wei Jia Bin Jia Xiaoying Jia Yuhua Jiang Haobin Jiang Ziheng Jiang Kaihua Jiang Chengquan Jiang Jianpeng Jiao Xiaoran Jin Xing Jin Xunhao Lai Zheng Li Xiang Li Liyi Li Hongkai Li Zheng Li Shengxian Wan Ya Wang Yunshui Li Chenggang Li Niuniu Li Siyu Li Xi Li Xiao Li Aoyan Li Yuntao Li Nianning Liang Xinnian Liang Haibin Lin Weijian Lin Ye Lin Zhicheng Liu Guanlin Liu Guanlin Liu Chenxiao Liu Yan Liu Gaohong Liu Juncai Liu Chundian Liu Deyi Liu Kaibo Liu Siyao Liu Qi Liu Yongfei Liu Kang Liu Gan Liu Boyi Liu Rui Long Weiqiang Lou Chenwei Lou Xiang Luo Yao Luo Caiping Lv Heyang Lv Bole Ma Qianli Ma Hongzhi Ma Yiyuan Ma Jin Ma Wenchang Ma Tingting Ma Chen Mao Qiyang Min Zhe Nan Guanghan Ning Jinxiang Ou Haojie Pan Renming Pang Yanghua Peng Tao Peng Lihua Qian Lihua Qian Mu Qiao Meng Qu Cheng Ren Hongbin Ren Yong Shan Wei Shen Ke Shen Kai Shen Guangming Sheng Jinlong Shi Wenlei Shi Guang Shi Shuai Shuai Cao Yuxin Song Zuquan Song Jing Su Yifan Sun Tao Sun Zewei Sun Borui Wan Zihan Wang Xiaohui Wang Xi Wang Shuguang Wang Jun Wang Qinlong Wang Chenyuan Wang Shuai Wang Zihan Wang Changbao Wang Jiaqiang Wang Shihang Wang Xuwu Wang Zaiyuan Wang Yuxuan Wang Wenqi Wang Taiqing Wang Chengzhi Wei Houmin Wei Ziyun Wei Shufa Wei Zheng Wu Yonghui Wu Yangjun Wu Bohong Wu Shuang Wu Jingqiao Wu Ning Wu Shuangzhi Wu Jianmin Wu Chenguang Xi Fan Xia Yuqiao Xian Liang Xiang Boren Xiang Bowen Xiao Zhen Xiao Xia Xiao Yongsheng Xiao Chao Xin Shulin Xin Yuwen Xiong Jingjing Xu Ziwen Xu Chenyin Xu Jiayi Xu Yifan Xu Wei Xu Yufei Xu Shikun Xu Shipeng Yan Shen Yan Qingping Yang Xi Yang Tianhao Yang Yuehang Yang Yuan Yang Ximing Yang Zeyu Yang Guang Yang Yifan Yang Xuesong Yao Bairen Yi Fan Yin Jianian Yin Ziqiang Ying Xiangyu Yu Hongli Yu Song Yu Menghan Yu Huan Yu Siyu Yuan Jun Yuan Yutao Zeng Tianyang Zhan Zheng Zhang Yun Zhang Mofan Zhang Wang Zhang Ru Zhang Zhi Zhang Tianqi Zhang Xinyi Zhang Zhexi Zhang Sijun Zhang Wenqiang Zhang Xiangxiang Zhang Yongtao Zhang Yuyu Zhang Ge Zhang He Zhang Yue Zhang Renjie Zheng Ningxin Zheng Zhuolin Zheng Yaowei Zheng Chen Zheng Xiaoyun Zhi Wanjun Zhong Cheng Zhong Zheng Zhong Baoquan Zhong Xun Zhou Na Zhou Huan Zhou Hang Zhu Defa Zhu Wenjia Zhu and Lei Zuo. 2025. Seed1.5-Thinking: Advancing Superb Reasoning Models with Reinforcement Learning. arXiv:2504.13914 [cs.CL] https:\/\/arxiv.org\/abs\/2504.13914"},{"key":"e_1_3_2_1_11_1","volume-title":"Rethwisch","author":"Callister William D.","year":"2016","unstructured":"William D. Callister and David G. Rethwisch. 2016. Fundamentals of Materials Science and Engineering: An Integrated Approach (5 ed.). Wiley Global Education."},{"key":"e_1_3_2_1_12_1","unstructured":"William D Callister Jr and David G Rethwisch. 2020. Materials science and engineering: an introduction. John wiley & sons."},{"key":"e_1_3_2_1_13_1","unstructured":"Karl Cobbe Vineet Kosaraju Mohammad Bavarian Mark Chen Heewoo Jun Lukasz Kaiser Matthias Plappert Jerry Tworek Jacob Hilton Reiichiro Nakano et al. 2021. Training verifiers to solve math word problems. arXiv preprint arXiv:2110.14168 (2021)."},{"key":"e_1_3_2_1_14_1","unstructured":"DeepMind. 2024. AI achieves silver-medal standard solving International Mathematical Olympiad problems. https:\/\/deepmind.google\/blog\/ai-solves-imoproblems-at-silver-medal-level\/"},{"key":"e_1_3_2_1_15_1","unstructured":"DeepSeek-AI Aixin Liu Bei Feng Bing Xue Bingxuan Wang Bochao Wu Chengda Lu Chenggang Zhao Chengqi Deng Chenyu Zhang Chong Ruan Damai Dai Daya Guo Dejian Yang Deli Chen Dongjie Ji Erhang Li Fangyun Lin Fucong Dai Fuli Luo Guangbo Hao Guanting Chen Guowei Li H. Zhang Han Bao Hanwei Xu Haocheng Wang Haowei Zhang Honghui Ding Huajian Xin Huazuo Gao Hui Li Hui Qu J. L. Cai Jian Liang Jianzhong Guo Jiaqi Ni Jiashi Li Jiawei Wang Jin Chen Jingchang Chen Jingyang Yuan Junjie Qiu Junlong Li Junxiao Song Kai Dong Kai Hu Kaige Gao Kang Guan Kexin Huang Kuai Yu LeanWang Lecong Zhang Lei Xu Leyi Xia Liang Zhao Litong Wang Liyue Zhang Meng Li MiaojunWang Mingchuan Zhang Minghua Zhang Minghui Tang Mingming Li Ning Tian Panpan Huang PeiyiWang Peng Zhang Qiancheng Wang Qihao Zhu Qinyu Chen Qiushi Du R. J. Chen R. L. Jin Ruiqi Ge Ruisong Zhang Ruizhe Pan Runji Wang Runxin Xu Ruoyu Zhang Ruyi Chen S. S. Li Shanghao Lu Shangyan Zhou Shanhuang Chen Shaoqing Wu Shengfeng Ye Shengfeng Ye Shirong Ma Shiyu Wang Shuang Zhou Shuiping Yu Shunfeng Zhou Shuting Pan T. Wang Tao Yun Tian Pei Tianyu Sun W. L. Xiao Wangding Zeng Wanjia Zhao Wei An Wen Liu Wenfeng Liang Wenjun Gao Wenqin Yu Wentao Zhang X. Q. Li Xiangyue Jin Xianzu Wang Xiao Bi Xiaodong Liu Xiaohan Wang Xiaojin Shen Xiaokang Chen Xiaokang Zhang Xiaosha Chen Xiaotao Nie Xiaowen Sun Xiaoxiang Wang Xin Cheng Xin Liu Xin Xie Xingchao Liu Xingkai Yu Xinnan Song Xinxia Shan Xinyi Zhou Xinyu Yang Xinyuan Li Xuecheng Su Xuheng Lin Y. K. Li Y. Q. Wang Y. X. Wei Y. X. Zhu Yang Zhang Yanhong Xu Yanhong Xu Yanping Huang Yao Li Yao Zhao Yaofeng Sun Yaohui Li Yaohui Wang Yi Yu Yi Zheng Yichao Zhang Yifan Shi Yiliang Xiong Ying He Ying Tang Yishi Piao Yisong Wang Yixuan Tan Yiyang Ma Yiyuan Liu Yongqiang Guo Yu Wu Yuan Ou Yuchen Zhu Yuduan Wang Yue Gong Yuheng Zou Yujia He Yukun Zha Yunfan Xiong Yunxian Ma Yuting Yan Yuxiang Luo Yuxiang You Yuxuan Liu Yuyang Zhou Z. F.Wu Z. Z. Ren Zehui Ren Zhangli Sha Zhe Fu Zhean Xu Zhen Huang Zhen Zhang Zhenda Xie Zhengyan Zhang Zhewen Hao Zhibin Gou Zhicheng Ma Zhigang Yan Zhihong Shao Zhipeng Xu Zhiyu Wu Zhongyu Zhang Zhuoshu Li Zihui Gu Zijia Zhu Zijun Liu Zilin Li Ziwei Xie Ziyang Song Ziyi Gao and Zizheng Pan. 2025. DeepSeek-V3 Technical Report. arXiv:2412.19437 [cs.CL] https:\/\/arxiv.org\/abs\/2412.19437"},{"key":"e_1_3_2_1_16_1","volume-title":"HARDMath: A Benchmark Dataset for Challenging Problems in Applied Mathematics. arXiv preprint arXiv:2410.09988","author":"Fan Jingxuan","year":"2024","unstructured":"Jingxuan Fan, Sarah Martinson, Erik Y Wang, Kaylie Hausknecht, Jonah Brenner, Danxian Liu, Nianli Peng, CoreyWang, and Michael P Brenner. 2024. HARDMath: A Benchmark Dataset for Challenging Problems in Applied Mathematics. arXiv preprint arXiv:2410.09988 (2024)."},{"key":"e_1_3_2_1_17_1","volume-title":"Materials and process selection for engineering design","author":"Farag Mahmoud M","unstructured":"Mahmoud M Farag. 2020. Materials and process selection for engineering design. CRC press."},{"key":"e_1_3_2_1_18_1","unstructured":"Kehua Feng Xinyi Shen Weijie Wang Xiang Zhuang Yuqi Tang Qiang Zhang and Keyan Ding. 2025. SciKnowEval: Evaluating Multi-level Scientific Knowledge of Large Language Models. arXiv:2406.09098 [cs.CL] https:\/\/arxiv.org\/abs\/2406.09098"},{"key":"e_1_3_2_1_19_1","volume-title":"PHYSICS: Benchmarking Foundation Models on University- Level Physics Problem Solving. arXiv:2503.21821 [cs.AI] https:\/\/arxiv.org\/abs\/25 03.21821","author":"Feng Kaiyue","year":"2025","unstructured":"Kaiyue Feng, Yilun Zhao, Yixin Liu, Tianyu Yang, Chen Zhao, John Sous, and Arman Cohan. 2025. PHYSICS: Benchmarking Foundation Models on University- Level Physics Problem Solving. arXiv:2503.21821 [cs.AI] https:\/\/arxiv.org\/abs\/25 03.21821"},{"key":"e_1_3_2_1_20_1","unstructured":"Joel R Fried. 2014. Polymer science and technology. Pearson Education."},{"key":"e_1_3_2_1_21_1","volume-title":"Electronic, magnetic, and optical materials","author":"Fulay Pradeep","unstructured":"Pradeep Fulay and Jung-Kun Lee. 2016. Electronic, magnetic, and optical materials. Crc Press."},{"key":"e_1_3_2_1_22_1","unstructured":"Gemma Team Aishwarya Kamath Johan Ferret Shreya Pathak Nino Vieillard Ramona Merhej Sarah Perrin Tatiana Matejovicova Alexandre Ram\u00e9 Morgane Rivi\u00e8re Louis Rouillard Thomas Mesnard Geoffrey Cideron Jean bastien Grill Sabela Ramos Edouard Yvinec Michelle Casbon Etienne Pot Ivo Penchev Ga\u00ebl Liu Francesco Visin Kathleen Kenealy Lucas Beyer Xiaohai Zhai Anton Tsitsulin Robert Busa-Fekete Alex Feng Noveen Sachdeva Benjamin Coleman Yi Gao Basil Mustafa Iain Barr Emilio Parisotto David Tian Matan Eyal Colin Cherry Jan-Thorsten Peter Danila Sinopalnikov Surya Bhupatiraju Rishabh Agarwal Mehran Kazemi Dan Malkin Ravin Kumar David Vilar Idan Brusilovsky Jiaming Luo Andreas Steiner Abe Friesen Abhanshu Sharma Abheesht Sharma Adi Mayrav Gilady Adrian Goedeckemeyer Alaa Saade Alex Feng Alexander Kolesnikov Alexei Bendebury Alvin Abdagic Amit Vadi Andr\u00e1s Gy\u00f6rgy Andr\u00e9 Susano Pinto Anil Das Ankur Bapna Antoine Miech Antoine Yang Antonia Paterson Ashish Shenoy Ayan Chakrabarti Bilal Piot Bo Wu Bobak Shahriari Bryce Petrini Charlie Chen Charline Le Lan Christopher A. Choquette-Choo CJ Carey Cormac Brick Daniel Deutsch Danielle Eisenbud Dee Cattle Derek Cheng Dimitris Paparas Divyashree Shivakumar Sreepathihalli Doug Reid Dustin Tran Dustin Zelle Eric Noland Erwin Huizenga Eugene Kharitonov Frederick Liu Gagik Amirkhanyan Glenn Cameron Hadi Hashemi Hanna Klimczak-Pluci'ska Harman Singh Harsh Mehta Harshal Tushar Lehri Hussein Hazimeh Ian Ballantyne Idan Szpektor Ivan Nardini Jean Pouget- Abadie Jetha Chan Joe Stanton John Wieting Jonathan Lai Jordi Orbay Joseph Fernandez Josh Newlan Ju yeong Ji Jyotinder Singh Kat Black Kathy Yu Kevin Hui Kiran Vodrahalli Klaus Greff Linhai Qiu Marcella Valentine Marina Coelho Marvin Ritter Matt Hoffman Matthew Watson Mayank Chaturvedi Michael Moynihan Min Ma Nabila Babar Natasha Noy Nathan Byrd Nick Roy Nikola Momchev Nilay Chauhan Noveen Sachdeva Oskar Bunyan Pankil Botarda Paul Caron Paul Kishan Rubenstein Phil Culliton Philipp Schmid Pier Giuseppe Sessa Pingmei Xu Piotr Stanczyk Pouya Tafti Rakesh Shivanna Renjie Wu Renke Pan Reza Rokni Rob Willoughby Rohith Vallu Ryan Mullins Sammy Jerome Sara Smoot Sertan Girgin Shariq Iqbal Shashir Reddy Shruti Sheth Siim P\u00f5der Sijal Bhatnagar Sindhu Raghuram Panyam Sivan Eiger Susan Zhang Tianqi Liu Trevor Yacovone Tyler Liechty Uday Kalra Utku Evci Vedant Misra Vincent Roseberry Vlad Feinberg Vlad Kolesnikov Woohyun Han Woosuk Kwon Xi Chen Yinlam Chow Yuvein Zhu Zichuan Wei Zoltan Egyed Victor Cotruta Minh Giang Phoebe Kirk Anand Rao Kat Black Nabila Babar Jessica Lo Erica Moreira Luiz Gustavo Martins Omar Sanseviero Lucas Gonzalez Zach Gleicher Tris Warkentin Vahab Mirrokni Evan Senter Eli Collins Joelle Barral Zoubin Ghahramani Raia Hadsell Yossi Matias D. Sculley Slav Petrov Noah Fiedel Noam Shazeer Oriol Vinyals Jeff Dean Demis Hassabis Koray Kavukcuoglu Clement Farabet Elena Buchatskaya Jean-Baptiste Alayrac Rohan Anil Dmitry Lepikhin Sebastian Borgeaud Olivier Bachem Armand Joulin Alek Andreev Cassidy Hardin Robert Dadashi and L\u00e9onard Hussenot. 2025. Gemma 3 Technical Report. arXiv:2503.19786 [cs.CL] https:\/\/arxiv.org\/abs\/2503.19786"},{"key":"e_1_3_2_1_23_1","volume-title":"Jean-Stanislas Denain, Anson Ho, Emily de Oliveira Santos, et al.","author":"Glazer Elliot","year":"2024","unstructured":"Elliot Glazer, Ege Erdil, Tamay Besiroglu, Diego Chicharro, Evan Chen, Alex Gunning, Caroline Falkman Olsson, Jean-Stanislas Denain, Anson Ho, Emily de Oliveira Santos, et al. 2024. Frontiermath: A benchmark for evaluating advanced mathematical reasoning in ai. arXiv preprint arXiv:2411.04872 (2024)."},{"key":"e_1_3_2_1_24_1","unstructured":"Google DeepMind. 2025. Gemini 2.5: Our most intelligent AI model. https:\/\/bl og.google\/innovation-and-ai\/models-and-research\/google-deepmind\/geminimodel- thinking-updates-march-2025\/#gemini-2-5-thinking."},{"key":"e_1_3_2_1_25_1","volume-title":"Tora: A tool-integrated reasoning agent for mathematical problem solving. arXiv preprint arXiv:2309.17452","author":"Gou Zhibin","year":"2023","unstructured":"Zhibin Gou, Zhihong Shao, Yeyun Gong, Yelong Shen, Yujiu Yang, Minlie Huang, Nan Duan, and Weizhu Chen. 2023. Tora: A tool-integrated reasoning agent for mathematical problem solving. arXiv preprint arXiv:2309.17452 (2023)."},{"key":"e_1_3_2_1_26_1","unstructured":"Etash Guha Ryan Marten Sedrick Keh Negin Raoof Georgios Smyrnis Hritik Bansal Marianna Nezhurina Jean Mercat Trung Vu Zayne Sprague Ashima Suvarna Benjamin Feuer Liangyu Chen Zaid Khan Eric Frankel Sachin Grover Caroline Choi Niklas Muennighoff Shiye Su Wanjia Zhao John Yang Shreyas Pimpalgaonkar Kartik Sharma Charlie Cheng-Jie Ji Yichuan Deng Sarah Pratt Vivek Ramanujan Jon Saad-Falcon Jeffrey Li Achal Dave Alon Albalak Kushal Arora Blake Wulfe Chinmay Hegde Greg Durrett Sewoong Oh Mohit Bansal Saadia Gabriel Aditya Grover Kai-Wei Chang Vaishaal Shankar Aaron Gokaslan Mike A. Merrill Tatsunori Hashimoto Yejin Choi Jenia Jitsev Reinhard Heckel Maheswaran Sathiamoorthy Alexandros G. Dimakis and Ludwig Schmidt. 2025. OpenThoughts: Data Recipes for Reasoning Models. arXiv:2506.04178 [cs.LG] https:\/\/arxiv.org\/abs\/2506.04178"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-025-09422-z"},{"key":"e_1_3_2_1_28_1","volume-title":"Enhancing question answering on charts through effective pre-training tasks. arXiv preprint arXiv:2406.10085","author":"Gupta Ashim","year":"2024","unstructured":"Ashim Gupta, Vivek Gupta, Shuo Zhang, Yujie He, Ning Zhang, and Shalin Shah. 2024. Enhancing question answering on charts through effective pre-training tasks. arXiv preprint arXiv:2406.10085 (2024)."},{"key":"e_1_3_2_1_29_1","volume-title":"Junhao Shen, Jinyi Hu, Xu Han, Yujie Huang, Yuxiang Zhang, et al.","author":"He Chaoqun","year":"2024","unstructured":"Chaoqun He, Renjie Luo, Yuzhuo Bai, Shengding Hu, Zhen Leng Thai, Junhao Shen, Jinyi Hu, Xu Han, Yujie Huang, Yuxiang Zhang, et al. 2024. Olympiadbench: A challenging benchmark for promoting agi with olympiad-level bilingual multimodal scientific problems. arXiv preprint arXiv:2402.14008 (2024)."},{"key":"e_1_3_2_1_30_1","volume-title":"Measuring mathematical problem solving with the math dataset. arXiv preprint arXiv:2103.03874","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, and Jacob Steinhardt. 2021. Measuring mathematical problem solving with the math dataset. arXiv preprint arXiv:2103.03874 (2021)."},{"key":"e_1_3_2_1_31_1","volume-title":"Mechanical behavior of materials","author":"Hosford William F","unstructured":"William F Hosford. 2010. Mechanical behavior of materials. Cambridge university press."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1201\/b15858"},{"key":"e_1_3_2_1_33_1","volume-title":"AdamsWei Yu, Xinying Song, and Denny Zhou.","author":"Huang Jie","year":"2023","unstructured":"Jie Huang, Xinyun Chen, Swaroop Mishra, Huaixiu Steven Zheng, AdamsWei Yu, Xinying Song, and Denny Zhou. 2023. Large language models cannot self-correct reasoning yet. arXiv preprint arXiv:2310.01798 (2023)."},{"key":"e_1_3_2_1_34_1","unstructured":"Yuqing Huang Rongyang Zhang Xuesong He Xuyang Zhi Hao Wang Xin Li Feiyang Xu Deguang Liu Huadong Liang Yi Li et al. 2024. ChemEval: A Comprehensive Multi-Level Chemical Evaluation for Large Language Models. arXiv preprint arXiv:2409.13989 (2024)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-0607"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.52202\/075280-1723"},{"key":"e_1_3_2_1_37_1","volume-title":"AtmosSci- Bench: Evaluating the Recent Advance of Large Language Model for Atmospheric Science. arXiv preprint arXiv:2502.01159","author":"Li Chenyue","year":"2025","unstructured":"Chenyue Li, Wen Deng, Mengqian Lu, and Binhang Yuan. 2025. AtmosSci- Bench: Evaluating the Recent Advance of Large Language Model for Atmospheric Science. arXiv preprint arXiv:2502.01159 (2025)."},{"key":"e_1_3_2_1_38_1","volume-title":"Mathvista: Evaluating mathematical reasoning of foundation models in visual contexts. arXiv preprint arXiv:2310.02255","author":"Lu Pan","year":"2023","unstructured":"Pan Lu, Hritik Bansal, Tony Xia, Jiacheng Liu, Chunyuan Li, Hannaneh Hajishirzi, Hao Cheng, Kai-Wei Chang, Michel Galley, and Jianfeng Gao. 2023. Mathvista: Evaluating mathematical reasoning of foundation models in visual contexts. arXiv preprint arXiv:2310.02255 (2023)."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"Pan Lu Swaroop Mishra Tony Xia Liang Qiu Kai-Wei Chang Song-Chun Zhu Oyvind Tafjord Peter Clark and Ashwin Kalyan. 2022. Learn to Explain: Multimodal Reasoning via Thought Chains for Science Question Answering. arXiv:2209.09513 [cs.CL] https:\/\/arxiv.org\/abs\/2209.09513","DOI":"10.52202\/068431-0182"},{"key":"e_1_3_2_1_40_1","unstructured":"M-A-P Team Xinrun Du Yifan Yao Kaijing Ma Bingli Wang Tianyu Zheng King Zhu Minghao Liu Yiming Liang Xiaolong Jin Zhenlin Wei Chujie Zheng Kaixin Deng Shawn Gavin Shian Jia Sichao Jiang Yiyan Liao Rui Li Qinrui Li Sirun Li Yizhi Li Yunwen Li David Ma Yuansheng Ni Haoran Que QiyaoWang ZhoufutuWen SiweiWu Tyshawn Hsing Ming Xu Zhenzhu Yang Zekun Moore Wang Junting Zhou Yuelin Bai Xingyuan Bu Chenglin Cai Liang Chen Yifan Chen Chengtuo Cheng Tianhao Cheng Keyi Ding Siming Huang Yun Huang Yaoru Li Yizhe Li Zhaoqun Li Tianhao Liang Chengdong Lin Hongquan Lin Yinghao Ma Tianyang Pang Zhongyuan Peng Zifan Peng Qige Qi Shi Qiu Xingwei Qu Shanghaoran Quan Yizhou Tan Zili Wang Chenqing Wang Hao Wang YiyaWang YuboWang Jiajun Xu Kexin Yang Ruibin Yuan Yuanhao Yue Tianyang Zhan Chun Zhang Jinyang Zhang Xiyue Zhang Xingjian Zhang Yue Zhang Yongchi Zhao Xiangyu Zheng Chenghua Zhong Yang Gao Zhoujun Li Dayiheng Liu Qian Liu Tianyu Liu Shiwen Ni Junran Peng Yujia Qin Wenbo Su Guoyin Wang Shi Wang Jian Yang Min Yang Meng Cao Xiang Yue Zhaoxiang Zhang Wangchunshu Zhou Jiaheng Liu Qunshu Lin Wenhao Huang and Ge Zhang. 2025. SuperGPQA: Scaling LLM Evaluation across 285 Graduate Disciplines. arXiv:2502.14739 [cs.CL] https:\/\/arxiv.org\/abs\/2502.14739"},{"key":"e_1_3_2_1_41_1","volume-title":"CliBench: A Multifaceted and Multigranular Evaluation of Large Language Models for Clinical Decision Making. arXiv preprint arXiv:2406.09923","author":"Ma Mingyu Derek","year":"2024","unstructured":"Mingyu Derek Ma, Chenchen Ye, Yu Yan, XiaoxuanWang, Peipei Ping, Timothy S Chang, and Wei Wang. 2024. CliBench: A Multifaceted and Multigranular Evaluation of Large Language Models for Clinical Decision Making. arXiv preprint arXiv:2406.09923 (2024)."},{"key":"e_1_3_2_1_42_1","volume-title":"Sciagent: Tool-augmented language models for scientific reasoning. arXiv preprint arXiv:2402.11451","author":"Ma Yubo","year":"2024","unstructured":"Yubo Ma, Zhibin Gou, Junheng Hao, Ruochen Xu, Shuohang Wang, Liangming Pan, Yujiu Yang, Yixin Cao, Aixin Sun, Hany Awadalla, et al. 2024. Sciagent: Tool-augmented language models for scientific reasoning. arXiv preprint arXiv:2402.11451 (2024)."},{"key":"e_1_3_2_1_43_1","unstructured":"Pragati Shuddhodhan Meshram Swetha Karthikeyan Bhavya Bhavya and Suma Bhat. 2025. ElectroVizQA: How well do Multi-modal LLMs perform in Electronics Visual Question Answering? arXiv:2412.00102 [cs.CV] https:\/\/arxiv.org\/abs\/2412.00102"},{"key":"e_1_3_2_1_44_1","unstructured":"Meta AI. 2025. The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation. https:\/\/ai.meta.com\/blog\/llama-4-multimodalintelligence\/. Accessed: 2025-05-15."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/WACV45572.2020.9093523"},{"key":"e_1_3_2_1_46_1","volume-title":"Gsm-symbolic: Understanding the limitations of mathematical reasoning in large language models. arXiv preprint arXiv:2410.05229","author":"Mirzadeh Iman","year":"2024","unstructured":"Iman Mirzadeh, Keivan Alizadeh, Hooman Shahrokhi, Oncel Tuzel, Samy Bengio, and Mehrdad Farajtabar. 2024. Gsm-symbolic: Understanding the limitations of mathematical reasoning in large language models. arXiv preprint arXiv:2410.05229 (2024)."},{"key":"e_1_3_2_1_47_1","unstructured":"Mistral AI Team. 2025. Mistral OCR. https:\/\/mistral.ai\/news\/mistral-ocr"},{"key":"e_1_3_2_1_48_1","unstructured":"OpenAI. 2025. Introducing GPT-5. https:\/\/openai.com\/index\/introducing-gpt-5\/. Accessed: 2025-09-24."},{"key":"e_1_3_2_1_49_1","unstructured":"OpenAI. 2025. Introducing GPT-4.1 in the API. https:\/\/openai.com\/index\/gpt-4-1\/."},{"key":"e_1_3_2_1_50_1","unstructured":"OpenAI. 2025. Introducing OpenAI o3 and o4-mini. https:\/\/openai.com\/index\/introducing-o3-and-o4-mini\/."},{"key":"e_1_3_2_1_51_1","unstructured":"OpenAI. 2025. OpenAI o3-mini. https:\/\/openai.com\/index\/openai-o3-mini\/"},{"key":"e_1_3_2_1_52_1","unstructured":"Qwen An Yang Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chengyuan Li Dayiheng Liu Fei Huang Haoran Wei Huan Lin Jian Yang Jianhong Tu Jianwei Zhang Jianxin Yang Jiaxi Yang Jingren Zhou Junyang Lin Kai Dang Keming Lu Keqin Bao Kexin Yang Le Yu Mei Li Mingfeng Xue Pei Zhang Qin Zhu Rui Men Runji Lin Tianhao Li Tianyi Tang Tingyu Xia Xingzhang Ren Xuancheng Ren Yang Fan Yang Su Yichang Zhang Yu Wan Yuqiong Liu Zeyu Cui Zhenru Zhang and Zihan Qiu. 2025. Qwen2.5 Technical Report. arXiv:2412.15115 [cs.CL] https:\/\/arxiv.org\/abs\/2412.15115"},{"key":"e_1_3_2_1_53_1","volume-title":"First Conference on Language Modeling.","author":"Rein David","year":"2024","unstructured":"David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, and Samuel R Bowman. 2024. Gpqa: A graduate-level google-proof q&a benchmark. In First Conference on Language Modeling."},{"key":"e_1_3_2_1_54_1","volume-title":"Introduction to Materials Science for Engineers (8 ed.). Pearson","author":"Shackelford James F.","unstructured":"James F. Shackelford. 2015. Introduction to Materials Science for Engineers (8 ed.). Pearson, Upper Saddle River, NJ."},{"key":"e_1_3_2_1_55_1","volume-title":"CRC Materials Science and Engineering Handbook (4 ed.)","author":"Shackelford James F.","unstructured":"James F. Shackelford, Young-Hwan Han, Sukyoung Kim, and Se-Hun Kwon. 2016. CRC Materials Science and Engineering Handbook (4 ed.). CRC Press, Boca Raton, FL."},{"key":"e_1_3_2_1_56_1","volume-title":"Reflexion: Language Agents with Verbal Reinforcement Learning. arXiv:2303.11366 [cs.AI] https:\/\/arxiv.org\/abs\/2303.11366","author":"Shinn Noah","year":"2023","unstructured":"Noah Shinn, Federico Cassano, Edward Berman, Ashwin Gopinath, Karthik Narasimhan, and Shunyu Yao. 2023. Reflexion: Language Agents with Verbal Reinforcement Learning. arXiv:2303.11366 [cs.AI] https:\/\/arxiv.org\/abs\/2303.11366"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i17.29872"},{"key":"e_1_3_2_1_58_1","unstructured":"Kimi Team Yifan Bai Yiping Bao Y. Charles Cheng Chen Guanduo Chen Haiting Chen Huarong Chen Jiahao Chen Ningxin Chen Ruijue Chen Yanru Chen Yuankun Chen Yutian Chen Zhuofu Chen Jialei Cui Hao Ding Mengnan Dong Angang Du Chenzhuang Du Dikang Du Yulun Du Yu Fan Yichen Feng Kelin Fu Bofei Gao Chenxiao Gao Hongcheng Gao Peizhong Gao Tong Gao Yuyao Ge Shangyi Geng Qizheng Gu Xinran Gu Longyu Guan Haiqing Guo Jianhang Guo Xiaoru Hao Tianhong He Weiran He Wenyang He Yunjia He Chao Hong Hao Hu Yangyang Hu Zhenxing Hu Weixiao Huang Zhiqi Huang Zihao Huang Tao Jiang Zhejun Jiang Xinyi Jin Yongsheng Kang Guokun Lai Cheng Li Fang Li Haoyang Li Ming Li Wentao Li Yang Li Yanhao Li Yiwei Li Zhaowei Li Zheming Li Hongzhan Lin Xiaohan Lin Zongyu Lin Chengyin Liu Chenyu Liu Hongzhang Liu Jingyuan Liu Junqi Liu Liang Liu Shaowei Liu T. Y. Liu Tianwei Liu Weizhou Liu Yangyang Liu Yibo Liu Yiping Liu Yue Liu Zhengying Liu Enzhe Lu Haoyu Lu Lijun Lu Yashuo Luo Shengling Ma Xinyu Ma Yingwei Ma Shaoguang Mao Jie Mei Xin Men Yibo Miao Siyuan Pan Yebo Peng Ruoyu Qin Zeyu Qin Bowen Qu Zeyu Shang Lidong Shi Shengyuan Shi Feifan Song Jianlin Su Zhengyuan Su Lin Sui Xinjie Sun Flood Sung Yunpeng Tai Heyi Tang Jiawen Tao Qifeng Teng Chaoran Tian Chensi Wang DingluWang FengWang HailongWang HaimingWang JianzhouWang Jiaxing Wang Jinhong Wang Shengjie Wang Shuyi Wang Si Wang Xinyuan Wang Yao Wang Yejie Wang Yiqin Wang Yuxin Wang Yuzhi Wang Zhaoji Wang ZhengtaoWang ZhengtaoWang ZhexuWang ChuWei QianqianWei Haoning Wu Wenhao Wu Xingzhe Wu Yuxin Wu Chenjun Xiao Jin Xie Xiaotong Xie Weimin Xiong Boyu Xu Jinjing Xu L. H. Xu Lin Xu Suting Xu Weixin Xu Xinran Xu Yangchuan Xu Ziyao Xu Jing Xu Jing Xu Junjie Yan Yuzi Yan Hao Yang Xiaofei Yang Yi Yang Ying Yang Zhen Yang Zhilin Yang Zonghan Yang Haotian Yao Xingcheng Yao Wenjie Ye Zhuorui Ye Bohong Yin Longhui Yu Enming Yuan Hongbang Yuan Mengjie Yuan Siyu Yuan Haobing Zhan Dehao Zhang Hao Zhang Wanlu Zhang Xiaobin Zhang Yadong Zhang Yangkun Zhang Yichi Zhang Yizhi Zhang Yongting Zhang Yu Zhang Yutao Zhang Yutong Zhang Zheng Zhang Haotian Zhao Yikai Zhao Zijia Zhao Huabin Zheng Shaojie Zheng Longguang Zhong Jianren Zhou Xinyu Zhou Zaida Zhou Jinguo Zhu Zhen Zhu Weiyu Zhuang and Xinxing Zu. 2026. Kimi K2: Open Agentic Intelligence. arXiv:2507.20534 [cs.LG] https:\/\/arxiv.org\/abs\/2507.20534"},{"key":"e_1_3_2_1_59_1","volume-title":"Large language models should be used as scientific reasoning engines, not knowledge databases. Nature medicine 29, 12","author":"Truhn Daniel","year":"2023","unstructured":"Daniel Truhn, Jorge S Reis-Filho, and Jakob Nikolas Kather. 2023. Large language models should be used as scientific reasoning engines, not knowledge databases. Nature medicine 29, 12 (2023), 2983-2984."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"crossref","unstructured":"George Tsoukalas Jasper Lee John Jennings Jimmy Xin Michelle Ding Michael Jennings Amitayush Thakur and Swarat Chaudhuri. 2024. PutnamBench: Evaluating Neural Theorem-Provers on the Putnam Mathematical Competition. arXiv:2407.11214 [cs.AI] https:\/\/arxiv.org\/abs\/2407.11214","DOI":"10.52202\/079017-0368"},{"key":"e_1_3_2_1_61_1","volume-title":"Scibench: Evaluating college-level scientific problem-solving abilities of large language models. arXiv preprint arXiv:2307.10635","author":"Wang Xiaoxuan","year":"2023","unstructured":"Xiaoxuan Wang, Ziniu Hu, Pan Lu, Yanqiao Zhu, Jieyu Zhang, Satyen Subramaniam, Arjun R Loomba, Shichang Zhang, Yizhou Sun, and Wei Wang. 2023. Scibench: Evaluating college-level scientific problem-solving abilities of large language models. arXiv preprint arXiv:2307.10635 (2023)."},{"key":"e_1_3_2_1_62_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2023","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Brian Ichter, Fei Xia, Ed Chi, Quoc Le, and Denny Zhou. 2023. Chain-of-Thought Prompting Elicits Reasoning in Large Language Models. arXiv:2201.11903 [cs.CL] https:\/\/arxiv.org\/abs\/2201.11903"},{"key":"e_1_3_2_1_63_1","volume-title":"Crowdsourcing multiple choice science questions. arXiv preprint arXiv:1707.06209","author":"Welbl Johannes","year":"2017","unstructured":"Johannes Welbl, Nelson F Liu, and Matt Gardner. 2017. Crowdsourcing multiple choice science questions. arXiv preprint arXiv:1707.06209 (2017)."},{"key":"e_1_3_2_1_64_1","volume-title":"Benchmarking benchmark leakage in large language models. arXiv preprint arXiv:2404.18824","author":"Xu Ruijie","year":"2024","unstructured":"Ruijie Xu, Zengzhi Wang, Run-Ze Fan, and Pengfei Liu. 2024. Benchmarking benchmark leakage in large language models. arXiv preprint arXiv:2404.18824 (2024)."},{"key":"e_1_3_2_1_65_1","unstructured":"An Yang Anfeng Li Baosong Yang Beichen Zhang Binyuan Hui Bo Zheng Bowen Yu Chang Gao Chengen Huang Chenxu Lv Chujie Zheng Dayiheng Liu Fan Zhou Fei Huang Feng Hu Hao Ge Haoran Wei Huan Lin Jialong Tang Jian Yang Jianhong Tu Jianwei Zhang Jianxin Yang Jiaxi Yang Jing Zhou Jingren Zhou Junyang Lin Kai Dang Keqin Bao Kexin Yang Le Yu Lianghao Deng Mei Li Mingfeng Xue Mingze Li Pei Zhang Peng Wang Qin Zhu Rui Men Ruize Gao Shixuan Liu Shuang Luo Tianhao Li Tianyi Tang Wenbiao Yin Xingzhang Ren Xinyu Wang Xinyu Zhang Xuancheng Ren Yang Fan Yang Su Yichang Zhang Yinger Zhang Yu Wan Yuqiong Liu Zekun Wang Zeyu Cui Zhenru Zhang Zhipeng Zhou and Zihan Qiu. 2025. Qwen3 Technical Report. https:\/\/arxiv.org\/abs\/2505.09388. arXiv:2505.09388 [cs.CL]."},{"key":"e_1_3_2_1_66_1","unstructured":"An Yang Beichen Zhang Binyuan Hui Bofei Gao Bowen Yu Chengpeng Li Dayiheng Liu Jianhong Tu Jingren Zhou Junyang Lin et al. 2024. Qwen2. 5- math technical report: Toward mathematical expert model via self-improvement. arXiv preprint arXiv:2409.12122 (2024)."},{"key":"e_1_3_2_1_67_1","volume-title":"Demystifying Long Chain-of-Thought Reasoning in LLMs. arXiv preprint arXiv:2502.03373","author":"Yeo Edward","year":"2025","unstructured":"Edward Yeo, Yuxuan Tong, Morry Niu, Graham Neubig, and Xiang Yue. 2025. Demystifying Long Chain-of-Thought Reasoning in LLMs. arXiv preprint arXiv:2502.03373 (2025)."},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52733.2024.00913"},{"key":"e_1_3_2_1_69_1","volume-title":"Mmmu-pro: A more robust multi-discipline multimodal understanding benchmark. arXiv preprint arXiv:2409.02813","author":"Yue Xiang","year":"2024","unstructured":"Xiang Yue, Tianyu Zheng, Yuansheng Ni, Yubo Wang, Kai Zhang, Shengbang Tong, Yuxuan Sun, Botao Yu, Ge Zhang, Huan Sun, et al. 2024. Mmmu-pro: A more robust multi-discipline multimodal understanding benchmark. arXiv preprint arXiv:2409.02813 (2024)."},{"key":"e_1_3_2_1_70_1","unstructured":"Mohd Zaki Jayadeva Mausam and N. M. Anoop Krishnan. 2023. MaScQA: A Question Answering Dataset for Investigating Materials Science Knowledge of Large Language Models. arXiv:2308.09115 [cs.CL] https:\/\/arxiv.org\/abs\/2308.09115"},{"key":"e_1_3_2_1_71_1","volume-title":"Lingling Zhang, and Jun Liu.","author":"Zhang Xinyu","year":"2025","unstructured":"Xinyu Zhang, Yuxuan Dong, Yanrui Wu, Jiaxing Huang, Chengyou Jia, Basura Fernando, Mike Zheng Shou, Lingling Zhang, and Jun Liu. 2025. Physreason: A comprehensive benchmark towards physics-based reasoning. arXiv preprint arXiv:2502.12054 (2025)."}],"event":{"name":"KDD '26: The 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Jeju Island Republic of Korea","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2"],"original-title":[],"deposited":{"date-parts":[[2026,8,7]],"date-time":"2026-08-07T01:31:05Z","timestamp":1786066265000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3770855.3818888"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8,8]]},"references-count":71,"alternative-id":["10.1145\/3770855.3818888","10.1145\/3770855"],"URL":"https:\/\/doi.org\/10.1145\/3770855.3818888","relation":{},"subject":[],"published":{"date-parts":[[2026,8,8]]},"assertion":[{"value":"2026-08-08","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}