{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T05:02:49Z","timestamp":1750309369612,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":14,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,3,31]],"date-time":"2025-03-31T00:00:00Z","timestamp":1743379200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,3,31]]},"DOI":"10.1145\/3672608.3707736","type":"proceedings-article","created":{"date-parts":[[2025,5,14]],"date-time":"2025-05-14T18:26:21Z","timestamp":1747247181000},"page":"48-53","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Assessing the Real-World Impact of Disagreement Between Human Graders and LLMs"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-6587-1527","authenticated-orcid":false,"given":"Sebastian","family":"Speiser","sequence":"first","affiliation":[{"name":"Hochschule f\u00fcr Technik, Stuttgart, Germany"}]}],"member":"320","published-online":{"date-parts":[[2025,5,14]]},"reference":[{"doi-asserted-by":"publisher","key":"e_1_3_2_1_1_1","DOI":"10.1111\/jcal.12717"},{"key":"e_1_3_2_1_2_1","volume-title":"Strategies for Deploying Unreliable AI Graders in High-Transparency High-Stakes Exams. Int. Conf. on Artificial Intelligence in Education","author":"Azad Sushmita","year":"2020","unstructured":"Sushmita Azad, Binglin Chen, Maxwell Fowler, Matthew West, and Craig Zilles. 2020. Strategies for Deploying Unreliable AI Graders in High-Transparency High-Stakes Exams. Int. Conf. on Artificial Intelligence in Education (2020)."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_3_1","DOI":"10.1007\/s40593-014-0026-8"},{"key":"e_1_3_2_1_4_1","volume-title":"Automatic Short Answer Grading for Finnish with ChatGPT. AAAI Conference on Artificial Intelligence","author":"Chang Li-Hsin","year":"2024","unstructured":"Li-Hsin Chang and Filip Ginter. 2024. Automatic Short Answer Grading for Finnish with ChatGPT. AAAI Conference on Artificial Intelligence (2024)."},{"key":"e_1_3_2_1_5_1","volume-title":"Automatic Evaluation of Open-Ended Questions for Online Learning. A Systematic Mapping. Studies in Educational Evaluation 77","author":"del Gobbo Emiliano","year":"2023","unstructured":"Emiliano del Gobbo, Alfonso Guarino, Barbara Cafarelli, Luca Grilli, and Pierpaolo Limone. 2023. Automatic Evaluation of Open-Ended Questions for Online Learning. A Systematic Mapping. Studies in Educational Evaluation 77 (2023)."},{"key":"e_1_3_2_1_6_1","volume-title":"Michael Granitzer, and Maximilian Sailer.","author":"Hackl Veronika","year":"2023","unstructured":"Veronika Hackl, Alexandra Elena M\u00fcller, Michael Granitzer, and Maximilian Sailer. 2023. Is GPT-4 a Reliable Rater? Evaluating Consistency in GPT-4 Text Ratings. Frontiers in Education 8 (2023). arXiv:2308.02575 [cs]"},{"unstructured":"Albert Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra Singh Chaplot Diego de las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier L\u00e9lio Renard Lavaud Marie-Anne Lachaux Pierre Stock Teven Le Scao Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William El Sayed. 2023. Mistral 7B. arXiv:2310.06825 (2023). arXiv:2310.06825 [cs]","key":"e_1_3_2_1_7_1"},{"key":"e_1_3_2_1_8_1","volume-title":"Work Smart - Reducing Effort in Short-Answer Grading. NLP4CALL","author":"Mieskes Margot","year":"2018","unstructured":"Margot Mieskes and Ulrike Pado. 2018. Work Smart - Reducing Effort in Short-Answer Grading. NLP4CALL (2018)."},{"key":"e_1_3_2_1_9_1","volume-title":"Int. Conf. on Computational Linguistics (COLING).","author":"Pad\u00f3 Ulrike","year":"2016","unstructured":"Ulrike Pad\u00f3. 2016. Get Semantic With Me! The Usefulness of Different Feature Types for Short-Answer Grading. In Int. Conf. on Computational Linguistics (COLING)."},{"key":"e_1_3_2_1_10_1","volume-title":"International Journal of Artificial Intelligence in Education","author":"Pad\u00f3 Ulrike","year":"2023","unstructured":"Ulrike Pad\u00f3, Yunus Eryilmaz, and Larissa Kirschner. 2023. Short-Answer Grading for German: Addressing the Challenges. International Journal of Artificial Intelligence in Education (2023)."},{"volume-title":"4th Workshop on NLP for computer-assisted language learning.","author":"Pad\u00f3 U.","unstructured":"U. Pad\u00f3 and C. Kiefer. 2015. Short Answer Grading: When Sorting Helps and When It Doesn't. In 4th Workshop on NLP for computer-assisted language learning.","key":"e_1_3_2_1_11_1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_12_1","DOI":"10.1007\/s10579-021-09547-3"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_13_1","DOI":"10.1109\/TLT.2023.3253071"},{"doi-asserted-by":"crossref","unstructured":"Johannes Schneider Bernd Schenk Christina Niklaus and Michaelis Vlachos. 2023. Towards LLM-based Autograding for Short Textual Answers. (2023). arXiv:2309.11508","key":"e_1_3_2_1_14_1","DOI":"10.5220\/0012552200003693"}],"event":{"sponsor":["SIGAPP ACM Special Interest Group on Applied Computing"],"acronym":"SAC '25","name":"SAC '25: 40th ACM\/SIGAPP Symposium on Applied Computing","location":"Catania International Airport Catania Italy"},"container-title":["Proceedings of the 40th ACM\/SIGAPP Symposium on Applied Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3672608.3707736","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3672608.3707736","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:06:14Z","timestamp":1750291574000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3672608.3707736"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,3,31]]},"references-count":14,"alternative-id":["10.1145\/3672608.3707736","10.1145\/3672608"],"URL":"https:\/\/doi.org\/10.1145\/3672608.3707736","relation":{},"subject":[],"published":{"date-parts":[[2025,3,31]]},"assertion":[{"value":"2025-05-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}