{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T00:41:46Z","timestamp":1768351306152,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":18,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,8,4]],"date-time":"2023-08-04T00:00:00Z","timestamp":1691107200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,8,6]]},"DOI":"10.1145\/3580305.3599827","type":"proceedings-article","created":{"date-parts":[[2023,8,4]],"date-time":"2023-08-04T18:13:58Z","timestamp":1691172838000},"page":"3947-3955","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["From Human Days to Machine Seconds: Automatically Answering and Generating Machine Learning Final Exams"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9797-3885","authenticated-orcid":false,"given":"Iddo","family":"Drori","sequence":"first","affiliation":[{"name":"MIT, Columbia University, Boston University, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-2209-6572","authenticated-orcid":false,"given":"Sarah J.","family":"Zhang","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6387-8780","authenticated-orcid":false,"given":"Reece","family":"Shuttleworth","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-8791-6680","authenticated-orcid":false,"given":"Sarah","family":"Zhang","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-7442-6026","authenticated-orcid":false,"given":"Keith","family":"Tyser","sequence":"additional","affiliation":[{"name":"Boston University, Boston, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-8733-3124","authenticated-orcid":false,"given":"Zad","family":"Chin","sequence":"additional","affiliation":[{"name":"Harvard University, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-2960-4715","authenticated-orcid":false,"given":"Pedro","family":"Lantigua","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-0258-8692","authenticated-orcid":false,"given":"Saisamrit","family":"Surbehera","sequence":"additional","affiliation":[{"name":"Columbia University, New York, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-9580-572X","authenticated-orcid":false,"given":"Gregory","family":"Hunter","sequence":"additional","affiliation":[{"name":"Columbia University, New York, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-5440-2566","authenticated-orcid":false,"given":"Derek","family":"Austin","sequence":"additional","affiliation":[{"name":"Columbia University, New York, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-9109-4496","authenticated-orcid":false,"given":"Leonard","family":"Tang","sequence":"additional","affiliation":[{"name":"Harvard University, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7234-7001","authenticated-orcid":false,"given":"Yann","family":"Hicke","sequence":"additional","affiliation":[{"name":"Cornell University, Ithaca, NY, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-6263-2870","authenticated-orcid":false,"given":"Sage","family":"Simhon","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-4898-5633","authenticated-orcid":false,"given":"Sathwik","family":"Karnik","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5571-310X","authenticated-orcid":false,"given":"Darnell","family":"Granberry","sequence":"additional","affiliation":[{"name":"MIT, Cambridge, MA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3985-915X","authenticated-orcid":false,"given":"Madeleine","family":"Udell","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,8,4]]},"reference":[{"key":"e_1_3_2_2_1_1","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown Tom B","year":"2020","unstructured":"Tom B Brown , Benjamin Mann , Nick Ryder , Melanie Subbiah , Jared Kaplan , Prafulla Dhariwal , Arvind Neelakantan , Pranav Shyam , Girish Sastry , Amanda Askell , 2020 . Language models are few-shot learners . In Proceedings of Advances in Neural Information Processing Systems (NeurIPS) , Vol. 33. 1877 -- 1901 . Tom B Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al. 2020. Language models are few-shot learners. In Proceedings of Advances in Neural Information Processing Systems (NeurIPS), Vol. 33. 1877--1901.","journal-title":"Proceedings of Advances in Neural Information Processing Systems (NeurIPS)"},{"key":"e_1_3_2_2_2_1","unstructured":"Mark Chen et al. 2021. Evaluating Large Language Models Trained on Code. arXiv preprint arXiv:2107.03374 (2021). arxiv: 2107.03374  Mark Chen et al. 2021. Evaluating Large Language Models Trained on Code. arXiv preprint arXiv:2107.03374 (2021). arxiv: 2107.03374"},{"key":"e_1_3_2_2_3_1","volume-title":"Charles Sutton, Sebastian Gehrmann, et al.","author":"Chowdhery Aakanksha","year":"2022","unstructured":"Aakanksha Chowdhery , Sharan Narang , Jacob Devlin , Maarten Bosma , Gaurav Mishra , Adam Roberts , Paul Barham , Hyung Won Chung , Charles Sutton, Sebastian Gehrmann, et al. 2022 . PaLM : Scaling language modeling with pathways. arXiv preprint arXiv:2204.02311 (2022). Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung Won Chung, Charles Sutton, Sebastian Gehrmann, et al. 2022. PaLM: Scaling language modeling with pathways. arXiv preprint arXiv:2204.02311 (2022)."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2123433119"},{"key":"e_1_3_2_2_5_1","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR).","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks , Collin Burns , Steven Basart , Andy Zou , Mantas Mazeika , Dawn Song , and Jacob Steinhardt . 2021 . Measuring massive multitask language understanding . In Proceedings of the International Conference on Learning Representations (ICLR). Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. 2021. Measuring massive multitask language understanding. In Proceedings of the International Conference on Learning Representations (ICLR)."},{"key":"e_1_3_2_2_6_1","volume-title":"Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa.","author":"Kojima Takeshi","year":"2022","unstructured":"Takeshi Kojima , Shixiang Shane Gu , Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa. 2022 . Large Language Models are Zero-Shot Reasoners . arXiv preprint arXiv:2205.11916 (2022). Takeshi Kojima, Shixiang Shane Gu, Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa. 2022. Large Language Models are Zero-Shot Reasoners. arXiv preprint arXiv:2205.11916 (2022)."},{"key":"e_1_3_2_2_7_1","volume-title":"Agustin Dal Lago, et al","author":"Li Yujia","year":"2022","unstructured":"Yujia Li , David Choi , Junyoung Chung , Nate Kushman , Julian Schrittwieser , R\u00e9mi Leblond , Tom Eccles , James Keeling , Felix Gimeno , Agustin Dal Lago, et al . 2022 . Competition-level code generation with alphacode. arXiv preprint arXiv:2203.07814 (2022). Yujia Li, David Choi, Junyoung Chung, Nate Kushman, Julian Schrittwieser, R\u00e9mi Leblond, Tom Eccles, James Keeling, Felix Gimeno, Agustin Dal Lago, et al. 2022. Competition-level code generation with alphacode. arXiv preprint arXiv:2203.07814 (2022)."},{"key":"e_1_3_2_2_8_1","unstructured":"Mathpix. 2023. Mathpix Snip. https:\/\/mathpix.com\/  Mathpix. 2023. Mathpix Snip. https:\/\/mathpix.com\/"},{"key":"e_1_3_2_2_9_1","unstructured":"OpenAI. 2022. ChatGPT: Optimizing Language Models for Dialogue. (2022).  OpenAI. 2022. ChatGPT: Optimizing Language Models for Dialogue. (2022)."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.202"},{"key":"e_1_3_2_2_11_1","unstructured":"Jack W Rae Sebastian Borgeaud Trevor Cai Katie Millican Jordan Hoffmann Francis Song John Aslanides Sarah Henderson Roman Ring Susannah Young etal 2021. Scaling language models: Methods analysis & insights from training gopher. arXiv preprint arXiv:2112.11446 (2021).  Jack W Rae Sebastian Borgeaud Trevor Cai Katie Millican Jordan Hoffmann Francis Song John Aslanides Sarah Henderson Roman Ring Susannah Young et al. 2021. Scaling language models: Methods analysis & insights from training gopher. arXiv preprint arXiv:2112.11446 (2021)."},{"key":"e_1_3_2_2_12_1","unstructured":"Teven Le Scao Angela Fan Christopher Akiki Ellie Pavlick Suzana Ili\u0107 Daniel Hesslow Roman Castagn\u00e9 Alexandra Sasha Luccioni Fran\u00e7ois Yvon Matthias Gall\u00e9 Jonathan Tow Alexander M. Rush Stella Biderman Albert Webson Pawan Sasanka Ammanamanchi Thomas Wang Beno\u00eet Sagot Niklas Muennighoff Albert Villanova del Moral Olatunji Ruwase Rachel Bawden Stas Bekman Angelina McMillan-Major Iz Beltagy Huu Nguyen Lucile Saulnier Samson Tan Pedro Ortiz Suarez Victor Sanh Hugo Lauren\u00e7on Yacine Jernite Julien Launay Margaret Mitchell Colin Raffel Aaron Gokaslan Adi Simhi Aitor Soroa Alham Fikri Aji Amit Alfassy Anna Rogers Ariel Kreisberg Nitzav Canwen Xu Chenghao Mou Chris Emezue Christopher Klamm Colin Leong Daniel van Strien David Ifeoluwa Adelani et al. 2022. BLOOM: A 176B-Parameter Open-Access Multilingual Language Model. arXiv preprint arXiv:2211.05100 (2022).  Teven Le Scao Angela Fan Christopher Akiki Ellie Pavlick Suzana Ili\u0107 Daniel Hesslow Roman Castagn\u00e9 Alexandra Sasha Luccioni Fran\u00e7ois Yvon Matthias Gall\u00e9 Jonathan Tow Alexander M. Rush Stella Biderman Albert Webson Pawan Sasanka Ammanamanchi Thomas Wang Beno\u00eet Sagot Niklas Muennighoff Albert Villanova del Moral Olatunji Ruwase Rachel Bawden Stas Bekman Angelina McMillan-Major Iz Beltagy Huu Nguyen Lucile Saulnier Samson Tan Pedro Ortiz Suarez Victor Sanh Hugo Lauren\u00e7on Yacine Jernite Julien Launay Margaret Mitchell Colin Raffel Aaron Gokaslan Adi Simhi Aitor Soroa Alham Fikri Aji Amit Alfassy Anna Rogers Ariel Kreisberg Nitzav Canwen Xu Chenghao Mou Chris Emezue Christopher Klamm Colin Leong Daniel van Strien David Ifeoluwa Adelani et al. 2022. BLOOM: A 176B-Parameter Open-Access Multilingual Language Model. arXiv preprint arXiv:2211.05100 (2022)."},{"key":"e_1_3_2_2_13_1","volume-title":"Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al.","author":"Srivastava Aarohi","year":"2022","unstructured":"Aarohi Srivastava , Abhinav Rastogi , Abhishek Rao , Abu Awal Md Shoeb , Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al. 2022 . Beyond the Imitation Game : Quantifying and extrapolating the capabilities of language models. arXiv preprint arXiv:2206.04615 (2022). Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R Brown, Adam Santoro, Aditya Gupta, Adri\u00e0 Garriga-Alonso, et al. 2022. Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models. arXiv preprint arXiv:2206.04615 (2022)."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-11647-6_127"},{"key":"e_1_3_2_2_15_1","volume-title":"Proceedings of the Asian Conference on Machine Learning (ACML). 470--485","author":"Tran Sunny","year":"2021","unstructured":"Sunny Tran , Pranav Krishna , Ishan Pakuwal , Prabhakar Kafle , Nikhil Singh , Jayson Lynch , and Iddo Drori . 2021 . Solving Machine Learning Problems . In Proceedings of the Asian Conference on Machine Learning (ACML). 470--485 . Sunny Tran, Pranav Krishna, Ishan Pakuwal, Prabhakar Kafle, Nikhil Singh, Jayson Lynch, and Iddo Drori. 2021. Solving Machine Learning Problems. In Proceedings of the Asian Conference on Machine Learning (ACML). 470--485."},{"key":"e_1_3_2_2_16_1","volume-title":"Chi, and Denny Zhou","author":"Wang Xuezhi","year":"2022","unstructured":"Xuezhi Wang , Jason Wei , Dale Schuurmans , Quoc Le , Ed Chi, and Denny Zhou . 2022 . Self-consistency improves chain of thought reasoning in language models. arXiv preprint arXiv:2203.11171 (2022). Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc Le, Ed Chi, and Denny Zhou. 2022. Self-consistency improves chain of thought reasoning in language models. arXiv preprint arXiv:2203.11171 (2022)."},{"key":"e_1_3_2_2_17_1","volume-title":"Chi, Quoc Le, and Denny Zhou","author":"Wei Jason","year":"2022","unstructured":"Jason Wei , Xuezhi Wang , Dale Schuurmans , Maarten Bosma , Ed Chi, Quoc Le, and Denny Zhou . 2022 . Chain of thought prompting elicits reasoning in large language models. arXiv preprint arXiv:2201.11903 (2022). Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Ed Chi, Quoc Le, and Denny Zhou. 2022. Chain of thought prompting elicits reasoning in large language models. arXiv preprint arXiv:2201.11903 (2022)."},{"key":"e_1_3_2_2_18_1","volume-title":"Chi","author":"Zhou Denny","year":"2022","unstructured":"Denny Zhou , Nathanael Sch\u00e4rli , Le Hou , Jason Wei , Nathan Scales , Xuezhi Wang , Dale Schuurmans , Olivier Bousquet , Quoc Le , and Ed Chi . 2022 . Least-to-Most Prompting Enables Complex Reasoning in Large Language Models . arXiv preprint arXiv:2205.10625 (2022). Denny Zhou, Nathanael Sch\u00e4rli, Le Hou, Jason Wei, Nathan Scales, Xuezhi Wang, Dale Schuurmans, Olivier Bousquet, Quoc Le, and Ed Chi. 2022. Least-to-Most Prompting Enables Complex Reasoning in Large Language Models. arXiv preprint arXiv:2205.10625 (2022)."}],"event":{"name":"KDD '23: The 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Long Beach CA USA","acronym":"KDD '23","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3580305.3599827","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3580305.3599827","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:49:23Z","timestamp":1750182563000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3580305.3599827"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,4]]},"references-count":18,"alternative-id":["10.1145\/3580305.3599827","10.1145\/3580305"],"URL":"https:\/\/doi.org\/10.1145\/3580305.3599827","relation":{},"subject":[],"published":{"date-parts":[[2023,8,4]]},"assertion":[{"value":"2023-08-04","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}