{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,2]],"date-time":"2025-08-02T19:01:41Z","timestamp":1754161301914,"version":"3.41.2"},"publisher-location":"New York, NY, USA","reference-count":32,"publisher":"ACM","funder":[{"DOI":"10.13039\/501100000646","name":"Japan Society for the Promotion of Science","doi-asserted-by":"publisher","award":["24K20905"],"award-info":[{"award-number":["24K20905"]}],"id":[{"id":"10.13039\/501100000646","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004298","name":"Secom Science and Technology Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004298","id-type":"DOI","asserted-by":"publisher"}]},{"name":"JST AIP Acceleration Research","award":["JPMJCR25U7"],"award-info":[{"award-number":["JPMJCR25U7"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,6,23]]},"DOI":"10.1145\/3696630.3727228","type":"proceedings-article","created":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:09:27Z","timestamp":1753729767000},"page":"711-721","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["An Empirical Study of the Error Characteristics in an Online Judge System"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-6221-4281","authenticated-orcid":false,"given":"Shota","family":"Shimizu","sequence":"first","affiliation":[{"name":"Automated Software Engineering Lab, Ritsumeikan University, Osaka, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-8777-619X","authenticated-orcid":false,"given":"Erina","family":"Makihara","sequence":"additional","affiliation":[{"name":"Automated Software Engineering Lab, Ritsumeikan University, Osaka, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4910-1729","authenticated-orcid":false,"given":"Norihiro","family":"Yoshida","sequence":"additional","affiliation":[{"name":"Automated Software Engineering Lab, Ritsumeikan University, Osaka, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,7,28]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 51st ACM Technical Symposium on Computer Science Education (SIGSCE). 340\u2013345","author":"Albrecht Ella","year":"2020","unstructured":"Ella Albrecht and Jens Grabowski. 2020. Sometimes It's Just Sloppiness - Studying Students' Programming Errors and Misconceptions. In Proceedings of the 51st ACM Technical Symposium on Computer Science Education (SIGSCE). 340\u2013345."},{"key":"e_1_3_2_1_2_1","volume-title":"Proceedings of the Working Group Reports on Innovation and Technology in Computer Science Education (ITiCSE). 177\u2013210","author":"Becker Brett A.","year":"2019","unstructured":"Brett A. Becker, Paul Denny, Raymond Pettit, Durell Bouchard, Dennis J. Bouvier, Brian Harrington, Amir Kamil, Amey Karkare, Chris McDonald, Peter-Michael Osera, Janice L. Pearce, and James Prather. 2019. Compiler Error Messages Considered Unhelpful: The Landscape of Text-Based Programming Error Message Research. In Proceedings of the Working Group Reports on Innovation and Technology in Computer Science Education (ITiCSE). 177\u2013210."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3011286.3011301","article-title":"Why is programming so difficult to learn?: Patterns of Difficulties Related to Programming Learning Mid-Stage","volume":"41","author":"Bosse Yorah","year":"2017","unstructured":"Yorah Bosse and Marco Aur\u00e9lio Gerosa. 2017. Why is programming so difficult to learn?: Patterns of Difficulties Related to Programming Learning Mid-Stage. ACM SIGSOFT Software Engineering Notes 41, 6 (2017), 1\u20136.","journal-title":"ACM SIGSOFT Software Engineering Notes"},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of International Conference on Software Engineering Education and Training (CSEE&T). IEEE, 1\u201310","author":"Cui Jialin","year":"2024","unstructured":"Jialin Cui, Runqiu Zhang, Fangtong Zhou, Ruochi Li, Yang Song, and Ed Gehringer. 2024. How Much Effort Do You Need to Expend on a Technical Interview? A Study of LeetCode Problem Solving Statistics. In Proceedings of International Conference on Software Engineering Education and Training (CSEE&T). IEEE, 1\u201310."},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of the 17th ACM Annual Conference on Innovation and Technology in Computer Science Education (ITiCSE). 75\u201380","author":"Denny Paul","year":"2012","unstructured":"Paul Denny, Andrew Luxton-Reilly, and Ewan Tempero. 2012. All syntax errors are not equal. In Proceedings of the 17th ACM Annual Conference on Innovation and Technology in Computer Science Education (ITiCSE). 75\u201380."},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the International Conference on Database Systems for Advanced Applications (DASFAA). 785\u2013789","author":"Dong Yu","year":"2000","unstructured":"Yu Dong, Jingyang Hou, and Xuesong Lu. 2000. An Intelligent Online Judge System for Programming Training. In Proceedings of the International Conference on Database Systems for Advanced Applications (DASFAA). 785\u2013789."},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 20th Australasian Computing Education Conference (ACE). 83\u201389","author":"Ettles Andrew","year":"2018","unstructured":"Andrew Ettles, Andrew Luxton-Reilly, and Paul Denny. 2018. Common logic errors made by novice programmers. In Proceedings of the 20th Australasian Computing Education Conference (ACE). 83\u201389."},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 2017 ACM SIGCSE Technical Symposium on Computer Science Education (SIGCSE). 267\u2013272","author":"Grover Shuchi","year":"2017","unstructured":"Shuchi Grover and Satabdi Basu. 2017. Measuring Student Learning in Introductory Block-Based Programming: Examining Misconceptions of Loops, Variables, and Boolean Logic. In Proceedings of the 2017 ACM SIGCSE Technical Symposium on Computer Science Education (SIGCSE). 267\u2013272."},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the 34th SIGCSE Technical Symposium on Computer Science Education. 153\u2013156","author":"Hristova Maria","year":"2003","unstructured":"Maria Hristova, Ananya Misra, Megan Rutter, and Rebecca Mercuri. 2003. Identifying and correcting Java programming errors for introductory computer science students. In Proceedings of the 34th SIGCSE Technical Symposium on Computer Science Education. 153\u2013156."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3379507","article-title":"Learning or Forgetting? A Dynamic Approach for Tracking the Knowledge Proficiency of Students","volume":"38","author":"Huang Zhenya","year":"2020","unstructured":"Zhenya Huang, Qi Liu, Yuying Chen, Le Wu, Keli Xiao, Enhong Chen, Haiping Ma, and Guoping Hu. 2020. Learning or Forgetting? A Dynamic Approach for Tracking the Knowledge Proficiency of Students. ACM Transactions on Information Systems 38, 2 (2020), 1\u201333.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the 10th Koli Calling International Conference on Computing Education Research. 86\u201393","author":"Ihantola Petri","year":"2010","unstructured":"Petri Ihantola, Tuukka Ahoniemi, Ville Karavirta, and Otto Sepp\u00e4l\u00e4. 2010. Review of recent systems for automatic assessment of programming assignments. In Proceedings of the 10th Koli Calling International Conference on Computing Education Research. 86\u201393."},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of the 2015 ITiCSE on Working Group Reports. 41\u201363","author":"Ihantola Petri","year":"2015","unstructured":"Petri Ihantola, Arto Vihavainen, Alireza Ahadi, Matthew Butler, J\u00fcrgen B\u00f6rstler, Stephen H. Edwards, Essi Isohanni, Ari Korhonen, Andrew Petersen, Kelly Rivers, Miguel \u00c1ngel Rubio, Judy Sheard, Bronius Skupas, Jaime Spacco, Claudia Szabo, and Daniel Toll. 2015. Educational Data Mining and Learning Analytics in Programming: Literature Review and Case Studies. In Proceedings of the 2015 ITiCSE on Working Group Reports. 41\u201363."},{"volume-title":"Proceedings of the International Conference on Awareness Science and Technology (iCAST). 313\u2013318","author":"Intisar Chowdhury Md","key":"e_1_3_2_1_13_1","unstructured":"Chowdhury Md Intisar and Yutaka Watanobe. [n. d.]. Classification of Online Judge Programmers based on Rule Extraction from Self Organizing Feature Map. In Proceedings of the International Conference on Awareness Science and Technology (iCAST). 313\u2013318."},{"volume-title":"Proceedings of the 41st ACM Technical Symposium on Computer Science Education (SIGCSE). 107\u2013111","author":"Kaczmarczyk Lisa C.","key":"e_1_3_2_1_14_1","unstructured":"Lisa C. Kaczmarczyk, Elizabeth R. Petrick, J. Philip East, and Geoffrey L. Herman. 2010. Identifying student misconceptions of programming. In Proceedings of the 41st ACM Technical Symposium on Computer Science Education (SIGCSE). 107\u2013111."},{"key":"e_1_3_2_1_15_1","volume-title":"Proceedings of the 14th International Conference on Natural Computation, Fuzzy Systems and Knowledge Discovery(ICNC-FSKD). 993\u20131000","author":"Liu Jianyu","year":"2018","unstructured":"Jianyu Liu, Shaohong Zhang, Zongbao Yang, Zhiqian Zhang, Jing Wang, and Xiaofei Xing. 2018. Online Judge System Topic Classification. In Proceedings of the 14th International Conference on Natural Computation, Fuzzy Systems and Knowledge Discovery(ICNC-FSKD). 993\u20131000."},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 32nd ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA). 334\u2013346","author":"Liu Kaibo","year":"2023","unstructured":"Kaibo Liu, Yudong Han, Jie M. Zhang, Zhenpeng Chen, Federica Sarro, Mark Harman, Gang Huang, and Yun Ma. 2023. Who Judges the Judge: An Empirical Study on Online Judge Tests. In Proceedings of the 32nd ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA). 334\u2013346."},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 6th International Conference on Computer Supported Education. 176\u2013183","author":"Mani Anaga","year":"2014","unstructured":"Anaga Mani, Divya Venkataramani, Jordi Petit, and Salvador Roura. 2014. Better Feedback for Educational Online Judges. In Proceedings of the 6th International Conference on Computer Supported Education. 176\u2013183."},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the 2024 9th International Conference on Distance Education and Learning (ICDEL). 278\u2013284","author":"Mi Huichao","year":"2024","unstructured":"Huichao Mi, Yichen Liu, Ruiqi Meng, Yike Tang, and Qiaorong Zhang. 2024. A Study of Learning Behavior Analysis and Student Achievement Based on an Online Judge System. In Proceedings of the 2024 9th International Conference on Distance Education and Learning (ICDEL). 278\u2013284."},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of the International Conference on Educational Data Mining (EDM). 473\u2013479","author":"Miao Dezhuang","year":"2020","unstructured":"Dezhuang Miao, Yu Dong, and Xuesong Lu. 2020. PIPE: Predicting Logical Programming Errors in Programming Exercises. In Proceedings of the International Conference on Educational Data Mining (EDM). 473\u2013479."},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the Neural Information Processing Systems (NeurIPS). 1\u201313","author":"Puri Ruchir","year":"2021","unstructured":"Ruchir Puri, David S Kung, Geert Janssen, Wei Zhang, Giacomo Domeniconi, Vladimir Zolotov, Julian Dolby, Jie Chen, Mihir Choudhury, Lindsey Decker, et al. 2021. Codenet: A large-scale ai for code dataset for learning a diversity of coding tasks. In Proceedings of the Neural Information Processing Systems (NeurIPS). 1\u201313."},{"key":"e_1_3_2_1_21_1","volume-title":"Uday Kiran Rage, and Keita Nakamura","author":"Mostafizer Rahman Md.","year":"2021","unstructured":"Md. Mostafizer Rahman, Yutaka Watanobe, Uday Kiran Rage, and Keita Nakamura. 2021. A Novel Rule-Based Online Judge Recommender System to Promote Computer Programming Education. In Advances and Trends in Artificial Intelligence. From Theory to Practice. 15\u201327."},{"key":"e_1_3_2_1_22_1","first-page":"131","article-title":"Competitive learning in informatics: The UVa online judge experience","volume":"2","author":"Revilla Miguel A","year":"2008","unstructured":"Miguel A Revilla, Shahriar Manzoor, and Rujia Liu. 2008. Competitive learning in informatics: The UVa online judge experience. Olympiads in Informatics 2, 10 (2008), 131\u2013148.","journal-title":"Olympiads in Informatics"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"crossref","first-page":"453","DOI":"10.1016\/j.chb.2013.04.001","article-title":"Student perception and usage of an automated programming assessment tool","volume":"31","author":"Rubio-S\u00e1nchez Manuel","year":"2014","unstructured":"Manuel Rubio-S\u00e1nchez, P\u00e4ivi Kinnunen, Crist\u00f3bal Pareja-Flores, and \u00c1ngel Vel\u00e1zquez-Iturbide. 2014. Student perception and usage of an automated programming assessment tool. Computers in Human Behavior 31 (2014), 453\u2013460.","journal-title":"Computers in Human Behavior"},{"key":"e_1_3_2_1_24_1","volume-title":"IDL-LTSOJ: Research and implementation of an intelligent online judge system utilizing DNN for defect localization. High-Confidence Computing","author":"Song Lihua","year":"2024","unstructured":"Lihua Song, Ying Han, Yufei Guo, and Chenying Cai. 2024. IDL-LTSOJ: Research and implementation of an intelligent online judge system utilizing DNN for defect localization. High-Confidence Computing (2024), 1\u201325."},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the 26th Australasian Computing Education Conference (ACE). 11\u201318","author":"MacNeil Stephen","year":"2024","unstructured":"Stephen MacNeil and Paul Denny and Andrew Tran and Juho Leinonen and Seth Bernstein and Arto Hellas and Sami Sarsa and Joanne Kim. 2024. Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models. In Proceedings of the 26th Australasian Computing Education Conference (ACE). 11\u201318."},{"key":"e_1_3_2_1_26_1","first-page":"157","article-title":"Online Education and Its Effective Practice","volume":"15","author":"Sun Anna","year":"2016","unstructured":"Anna Sun and Xiufang Chen. 2016. Online Education and Its Effective Practice: A Research Review. Journal of Information Technology Education: Research 15 (2016), 157\u2013190.","journal-title":"Journal of Information Technology Education: Research"},{"key":"e_1_3_2_1_27_1","first-page":"1","article-title":"How problem difficulty and order influence programming education outcomes in online judge systems","volume":"9","author":"Wang Jinshui","year":"2023","unstructured":"Jinshui Wang, Pengchen Lin, Zhengyi Tang, and Shuguang Chen. 2023. How problem difficulty and order influence programming education outcomes in online judge systems. Heliyon 9, 11 (2023), 1\u201314.","journal-title":"Heliyon"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3143560","article-title":"A Survey on Online Judge Systems and Their Applications","volume":"51","author":"Wasik Szymon","year":"2018","unstructured":"Szymon Wasik, Maciej Antczak, Jan Badura, Artur Laskowski, and Tomasz Sternal. 2018. A Survey on Online Judge Systems and Their Applications. ACM Computing Surveys (CSUR) 51, 1 (2018), 1\u201334.","journal-title":"ACM Computing Surveys (CSUR)"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","first-page":"917","DOI":"10.1142\/S0218194022500346","article-title":"Online Judge System: Requirements, Architecture, and Experiences","volume":"32","author":"Watanobe Yutaka","year":"2022","unstructured":"Yutaka Watanobe, Md. Mostafizer Rahman, Taku Matsumoto, Uday Kiran Rage, and Penugonda Ravikumar. 2022. Online Judge System: Requirements, Architecture, and Experiences. International Journal of Software Engineering and Knowledge Engineering 32, 06 (2022), 917\u2013946.","journal-title":"International Journal of Software Engineering and Knowledge Engineering"},{"key":"e_1_3_2_1_30_1","first-page":"1","article-title":"Learning Style Integrated Deep Reinforcement Learning Framework for Programming Problem Recommendation in Online Judge System","volume":"15","author":"Xu Yuhui","year":"2022","unstructured":"Yuhui Xu, Qin Ni, Shuang Liu, Yifei Mi, Yangze Yu, and Yujia Hao. 2022. Learning Style Integrated Deep Reinforcement Learning Framework for Programming Problem Recommendation in Online Judge System. International Journal of Computational Intelligence Systems 15, 114 (2022), 1\u201322.","journal-title":"International Journal of Computational Intelligence Systems"},{"key":"e_1_3_2_1_31_1","first-page":"334","article-title":"Automatically Learning Topics and Difficulty Levels of Problems in Online Judge Systems","volume":"36","author":"Zhao Wayne Xin","year":"2018","unstructured":"Wayne Xin Zhao, Wenhui Zhang, Yulan He, Xing Xie, and Ji-Rong Wen. 2018. Automatically Learning Topics and Difficulty Levels of Problems in Online Judge Systems. ACM Transactions on Information Systems 36 (2018), 334\u2013346.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of ACM Turing Celebration Conference - China. 9\u201314","author":"Zhou Wenju","year":"2018","unstructured":"Wenju Zhou, Yigong Pan, Yinghua Zhou, and Guangzhong Sun. 2018. The framework of a new online judge system for programming education. In Proceedings of ACM Turing Celebration Conference - China. 9\u201314."}],"event":{"name":"FSE Companion '25: 33rd ACM International Conference on the Foundations of Software Engineering","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"],"location":"Clarion Hotel Trondheim Trondheim Norway","acronym":"FSE Companion '25"},"container-title":["Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3696630.3727228","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T19:10:02Z","timestamp":1753729802000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3696630.3727228"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6,23]]},"references-count":32,"alternative-id":["10.1145\/3696630.3727228","10.1145\/3696630"],"URL":"https:\/\/doi.org\/10.1145\/3696630.3727228","relation":{},"subject":[],"published":{"date-parts":[[2025,6,23]]},"assertion":[{"value":"2025-07-28","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}