{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T03:48:55Z","timestamp":1791431335548,"version":"4.3.3"},"publisher-location":"New York, NY, USA","reference-count":63,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706598.3714051","type":"proceedings-article","created":{"date-parts":[[2025,4,23]],"date-time":"2025-04-23T23:24:33Z","timestamp":1745450673000},"page":"1-22","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":25,"title":["PromptHive: Bringing Subject Matter Experts Back to the Forefront with Collaborative Prompt Engineering for Educational Content Creation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9668-3384","authenticated-orcid":false,"given":"Mohi","family":"Reza","sequence":"first","affiliation":[{"name":"Computer Science, University of Toronto, Toronto, Ontario, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1341-5876","authenticated-orcid":false,"given":"Ioannis","family":"Anastasopoulos","sequence":"additional","affiliation":[{"name":"Berkeley School of Education, UC Berkeley, Berkeley, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-1705-3052","authenticated-orcid":false,"given":"Shreya","family":"Bhandari","sequence":"additional","affiliation":[{"name":"Electrical Engineering & Computer Science, UC Berkeley, Berkeley, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6016-7051","authenticated-orcid":false,"given":"Zachary A.","family":"Pardos","sequence":"additional","affiliation":[{"name":"Berkeley School of Education, UC Berkeley, Berkeley, California, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_3_2_2","doi-asserted-by":"crossref","unstructured":"Bram Adams and Foutse Khomh. 2020. The diversity crisis of software engineering for artificial intelligence. IEEE Software 37 5 (2020) 104\u2013108.","DOI":"10.1109\/MS.2020.2975075"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1007\/11774303_7"},{"key":"e_1_3_3_3_4_2","first-page":"27","volume-title":"iTextbooks@ AIED","author":"Anastasopoulos Ioannis","year":"2023","unstructured":"Ioannis Anastasopoulos. 2023. Exploring the Content Ecosystem of the First Open-source Adaptive Tutor and its Applications on Intelligent Textbooks.. In iTextbooks@ AIED. 27\u201336."},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"crossref","unstructured":"Alejandro\u00a0Barredo Arrieta Natalia D\u00edaz-Rodr\u00edguez Javier Del\u00a0Ser Adrien Bennetot Siham Tabik Alberto Barbado Salvador Garc\u00eda Sergio Gil-L\u00f3pez Daniel Molina Richard Benjamins et\u00a0al. 2020. Explainable Artificial Intelligence (XAI): Concepts taxonomies opportunities and challenges toward responsible AI. Information fusion 58 (2020) 82\u2013115.","DOI":"10.1016\/j.inffus.2019.12.012"},{"key":"e_1_3_3_3_6_2","doi-asserted-by":"publisher","unstructured":"Zahra Ashktorab Michael Desmond Josh Andres Michael Muller Narendra\u00a0Nath Joshi Michelle Brachman Aabhas Sharma Kristina Brimijoin Qian Pan Christine\u00a0T. Wolf Evelyn Duesterwald Casey Dugan Werner Geyer and Darrell Reimer. 2021. AI-Assisted Human Labeling: Batching for Efficiency without Overreliance. Proc. ACM Hum.-Comput. Interact. 5 CSCW1 Article 89 (apr 2021) 27\u00a0pages. 10.1145\/3449163","DOI":"10.1145\/3449163"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"crossref","unstructured":"Shreya Bhandari Yunting Liu Yerin Kwak and Zachary\u00a0A Pardos. 2024. Evaluating the Psychometric Properties of ChatGPT-generated Questions. Computers and Education: Artificial Intelligence (2024) 100284.","DOI":"10.1016\/j.caeai.2024.100284"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"crossref","unstructured":"Mohamed\u00a0Zulhilmi bin Mohamed Riyan Hidayat Nurain\u00a0Nabilah binti Suhaizi Muhamad Khairul\u00a0Hakim bin Mahmud Siti\u00a0Nurshafikah binti Baharuddin et\u00a0al. 2022. Artificial intelligence in mathematics education: A systematic literature review. International Electronic Journal of Mathematics Education 17 3 (2022) em0694.","DOI":"10.29333\/iejme\/12132"},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606725"},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"crossref","unstructured":"Virginia Braun and Victoria Clarke. 2019. Reflecting on reflexive thematic analysis. Qualitative research in sport exercise and health 11 4 (2019) 589\u2013597.","DOI":"10.1080\/2159676X.2019.1628806"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"crossref","unstructured":"Li Cheng Ethan Croteau Sami Baral Cristina Heffernan and Neil Heffernan. 2024. Facilitating student learning with a chatbot in an online math learning platform. Journal of Educational Computing Research 62 4 (2024) 907\u2013937.","DOI":"10.1177\/07356331241226592"},{"key":"e_1_3_3_3_12_2","unstructured":"Douglas Crockford. 2024. JSON: JavaScript Object Notation. https:\/\/www.json.org\/json-en.html Accessed: 2024-12-10."},{"key":"e_1_3_3_3_13_2","unstructured":"Zhiwei Fei Xiaoyu Shen Dawei Zhu Fengzhe Zhou Zhuo Han Songyang Zhang Kai Chen Zongwen Shen and Jidong Ge. 2023. Lawbench: Benchmarking legal knowledge of large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2309.16289 (2023)."},{"key":"e_1_3_3_3_14_2","doi-asserted-by":"crossref","unstructured":"Jakub Harasta Tereza Novotn\u00e1 and Jaromir Savelka. 2024. It Cannot Be Right If It Was Written by AI: On Lawyers\u2019 Preferences of Documents Perceived as Authored by an LLM vs a Human. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.06798 (2024).","DOI":"10.1007\/s10506-024-09422-w"},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1177\/154193120605000909"},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"crossref","unstructured":"Neil\u00a0T Heffernan and Cristina\u00a0Lindquist Heffernan. 2014. The ASSISTments ecosystem: Building a platform that brings scientists and teachers together for minimally invasive research on human learning and teaching. International Journal of Artificial Intelligence in Education 24 (2014) 470\u2013497.","DOI":"10.1007\/s40593-014-0024-x"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"crossref","unstructured":"Robert\u00a0R Hoffman Shane\u00a0T Mueller Gary Klein and Jordan Litman. 2023. Measures for explainable AI: explanation goodness user satisfaction mental models curiosity trust and human-AI performance. Front. Comput. Sci. 5 1096257 (2023).","DOI":"10.3389\/fcomp.2023.1096257"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"crossref","unstructured":"Ju\u00a0Yeon Jung Tom Steinberger John\u00a0L King and Mark\u00a0S Ackerman. 2022. How domain experts work with data: Situating data science in the practices and settings of craftwork. Proceedings of the ACM on human-computer interaction 6 CSCW1 (2022) 1\u201329.","DOI":"10.1145\/3512905"},{"key":"e_1_3_3_3_19_2","unstructured":"Sal Khan. 2023. How AI Could Save (Not Destroy) Education. TED Talk available at https:\/\/www.ted.com\/talks\/sal_khan_how_ai_could_save_not_destroy_education."},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3586183.3606833"},{"key":"e_1_3_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584078"},{"key":"e_1_3_3_3_22_2","unstructured":"Harsh Kumar Ilya Musabirov Mohi Reza Jiakai Shi Anastasia Kuzminykh Joseph\u00a0Jay Williams and Michael Liut. 2023. Impact of guidance and interaction strategies for LLM use on Learner Performance and perception. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2310.13712 (2023)."},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"crossref","unstructured":"Pawan Kumar and Manmohan Sharma. 2022. Data machine learning and human domain experts: none is better than their collaboration. International Journal of Human\u2013Computer Interaction 38 14 (2022) 1307\u20131320.","DOI":"10.1080\/10447318.2021.2002040"},{"key":"e_1_3_3_3_24_2","doi-asserted-by":"crossref","unstructured":"James\u00a0R Lewis. 2018. The system usability scale: past present and future. International Journal of Human\u2013Computer Interaction 34 7 (2018) 577\u2013590.","DOI":"10.1080\/10447318.2018.1455307"},{"key":"e_1_3_3_3_25_2","unstructured":"Junyou Li Qin Zhang Yangbin Yu Qiang Fu and Deheng Ye. 2024. More agents is all you need. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2402.05120 (2024)."},{"key":"e_1_3_3_3_26_2","unstructured":"Yunting Liu Shreya Bhandari and Zachary\u00a0A Pardos. 2024. Leveraging LLM-Respondents for Item Evaluation: a Psychometric Analysis. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.10899 (2024)."},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3657604.3662031"},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"crossref","unstructured":"Bertalan Mesk\u00f3 and Eric\u00a0J Topol. 2023. The imperative for regulatory oversight of large language models (or generative AI) in healthcare. NPJ digital medicine 6 1 (2023) 120.","DOI":"10.1038\/s41746-023-00873-0"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-42682-7_16"},{"key":"e_1_3_3_3_30_2","volume-title":"Mathematical Intelligence: What We Have that Machines Don\u2019t","author":"Mubeen Junaid","year":"2022","unstructured":"Junaid Mubeen. 2022. Mathematical Intelligence: What We Have that Machines Don\u2019t. Profile Books."},{"key":"e_1_3_3_3_31_2","unstructured":"National Council of Teachers of Mathematics. 2024. Artificial Intelligence and Mathematics Teaching. Available at https:\/\/www.nctm.org\/standards-and-positions\/Position-Statements\/Artificial-Intelligence-and-Mathematics-Teaching\/."},{"key":"e_1_3_3_3_32_2","doi-asserted-by":"crossref","unstructured":"Philip\u00a0M Newton and Maira Xiromeriti. 2023. ChatGPT performance on MCQ exams in higher education. A pragmatic scoping review. EdArXiv. February 21 (2023).","DOI":"10.35542\/osf.io\/sytu3"},{"key":"e_1_3_3_3_33_2","first-page":"278","volume-title":"European Conference on Technology Enhanced Learning","author":"Nguyen Huy\u00a0A","year":"2023","unstructured":"Huy\u00a0A Nguyen, Hayden Stec, Xinying Hou, Sarah Di, and Bruce\u00a0M McLaren. 2023. Evaluating chatgpt\u2019s decimal skills and feedback generation in a digital learning game. In European Conference on Technology Enhanced Learning. Springer, 278\u2013293."},{"key":"e_1_3_3_3_34_2","unstructured":"OATutor. 2024. OATutor: Open Adaptive Tutoring for Teachers. Available at https:\/\/www.oatutor.io\/teachers (accessed December 10 2024)."},{"key":"e_1_3_3_3_35_2","unstructured":"OpenAI. [n. d.]. https:\/\/platform.openai.com\/docs\/guides\/prompt-engineering\/strategy-split-complex-tasks-into-simpler-subtasks"},{"key":"e_1_3_3_3_36_2","unstructured":"OpenAI. 2024. Introducing Structured Outputs in the API. OpenAI (August 2024). https:\/\/openai.com\/index\/introducing-structured-outputs-in-the-api\/"},{"key":"e_1_3_3_3_37_2","unstructured":"OpenStax. [n. d.]. OpenStax College Algebra 2e. https:\/\/academia.stackexchange.com\/questions\/14010\/how-do-you-cite-a-github-repository"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"crossref","unstructured":"John\u00a0F Pane Beth\u00a0Ann Griffin Daniel\u00a0F McCaffrey and Rita Karam. 2014. Effectiveness of cognitive tutor algebra I at scale. Educational Evaluation and Policy Analysis 36 2 (2014) 127\u2013144.","DOI":"10.3102\/0162373713507480"},{"key":"e_1_3_3_3_39_2","unstructured":"Zachary\u00a0A. Pardos.2023. Open-source Adaptive Tutoring System (OATutor). https:\/\/github.com\/CAHLR\/OATutor-Content\/tree\/."},{"key":"e_1_3_3_3_40_2","unstructured":"Zachary\u00a0A Pardos and Shreya Bhandari. 2023. Learning gain differences between ChatGPT and human tutor generated algebra hints. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2302.06871 (2023)."},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"crossref","unstructured":"Zachary\u00a0A Pardos and Shreya Bhandari. 2024. ChatGPT-generated help produces learning gains equivalent to human tutor-authored help on mathematics skills. Plos one 19 5 (2024) e0304013.","DOI":"10.1371\/journal.pone.0304013"},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581574"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"crossref","unstructured":"Muhammad Raees Inge Meijerink Ioanna Lykourentzou Vassilis-Javed Khan and Konstantinos Papangelis. 2024. From explainable to interactive AI: A literature review on current trends in human-AI interaction. International Journal of Human-Computer Studies (2024) 103301.","DOI":"10.1016\/j.ijhcs.2024.103301"},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"crossref","unstructured":"Leena Razzaq Jozsef Patvarczki Shane\u00a0F Almeida Manasi Vartak Mingyu Feng Neil\u00a0T Heffernan and Kenneth\u00a0R Koedinger. 2009. The Assistment Builder: Supporting the life cycle of tutoring system content creation. IEEE Transactions on Learning Technologies 2 2 (2009) 157\u2013166.","DOI":"10.1109\/TLT.2009.23"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641899"},{"key":"e_1_3_3_3_46_2","doi-asserted-by":"crossref","unstructured":"Abel Salinas and Fred Morstatter. 2024. The butterfly effect of altering prompts: How small changes and jailbreaks affect large language model performance. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2401.03729 (2024).","DOI":"10.18653\/v1\/2024.findings-acl.275"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517578"},{"key":"e_1_3_3_3_48_2","doi-asserted-by":"publisher","DOI":"10.1145\/3636555.3636919"},{"key":"e_1_3_3_3_49_2","doi-asserted-by":"publisher","DOI":"10.21236\/ADA057655"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"crossref","unstructured":"Ben Shneiderman. 2020. Human-centered artificial intelligence: Reliable safe & trustworthy. International Journal of Human\u2013Computer Interaction 36 6 (2020) 495\u2013504.","DOI":"10.1080\/10447318.2020.1741118"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"publisher","DOI":"10.1145\/3538882.3542790"},{"key":"e_1_3_3_3_52_2","unstructured":"Joykirat Singh Akshay Nambi and Vibhav Vineet. 2024. Exposing the Achilles\u2019 Heel: Evaluating LLMs Ability to Handle Mistakes in Mathematical Reasoning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2406.10834 (2024)."},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.1093\/oso\/9780198824305.001.0001"},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"crossref","unstructured":"Daniel Stafford and Robert Flatley. 2018. Openstax. The Charleston Advisor 20 1 (2018) 48\u201351.","DOI":"10.5260\/chara.20.1.48"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642902"},{"key":"e_1_3_3_3_56_2","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124960"},{"key":"e_1_3_3_3_57_2","unstructured":"Ruochen Wang Sohyun An Minhao Cheng Tianyi Zhou Sung\u00a0Ju Hwang and Cho-Jui Hsieh. 2024. One Prompt is not Enough: Automated Construction of a Mixture-of-Expert Prompts. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2407.00256 (2024)."},{"key":"e_1_3_3_3_58_2","unstructured":"Xuezhi Wang Jason Wei Dale Schuurmans Quoc Le Ed Chi Sharan Narang Aakanksha Chowdhery and Denny Zhou. 2022. Self-consistency improves chain of thought reasoning in language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2203.11171 (2022)."},{"key":"e_1_3_3_3_59_2","unstructured":"Zijie\u00a0J Wang Aishwarya Chakravarthy David Munechika and Duen\u00a0Horng Chau. 2024. Wordflow: Social prompt engineering for large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2401.14447 (2024)."},{"key":"e_1_3_3_3_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376226"},{"key":"e_1_3_3_3_61_2","volume-title":"The math (s) fix: An education blueprint for the AI age","author":"Wolfram Conrad","year":"2020","unstructured":"Conrad Wolfram. 2020. The math (s) fix: An education blueprint for the AI age. Wolfram Media."},{"key":"e_1_3_3_3_62_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_3_63_2","doi-asserted-by":"crossref","unstructured":"Peng Zhang and Maged\u00a0N Kamel\u00a0Boulos. 2023. Generative AI in medicine and healthcare: promises opportunities and challenges. Future Internet 15 9 (2023) 286.","DOI":"10.3390\/fi15090286"},{"key":"e_1_3_3_3_64_2","doi-asserted-by":"crossref","unstructured":"Eric Zhou and Dokyun Lee. 2024. Generative artificial intelligence human creativity and art. PNAS nexus 3 3 (2024) pgae052.","DOI":"10.1093\/pnasnexus\/pgae052"}],"event":{"name":"CHI 2025: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3714051","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706598.3714051","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,7,9]],"date-time":"2025-07-09T14:47:32Z","timestamp":1752072452000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706598.3714051"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":63,"alternative-id":["10.1145\/3706598.3714051","10.1145\/3706598"],"URL":"https:\/\/doi.org\/10.1145\/3706598.3714051","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}