{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T01:01:38Z","timestamp":1779152498905,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":34,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,11,17]],"date-time":"2022-11-17T00:00:00Z","timestamp":1668643200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"NSF","award":["2030859"],"award-info":[{"award-number":["2030859"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,11,17]]},"DOI":"10.1145\/3564721.3564726","type":"proceedings-article","created":{"date-parts":[[2022,11,12]],"date-time":"2022-11-12T23:17:42Z","timestamp":1668295062000},"page":"1-7","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Making Hay from Wheats: A Classsourcing Method to Identify Misconceptions"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7936-8147","authenticated-orcid":false,"given":"Siddhartha","family":"Prasad","sequence":"first","affiliation":[{"name":"Brown University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7078-9287","authenticated-orcid":false,"given":"Ben","family":"Greenman","sequence":"additional","affiliation":[{"name":"Brown University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9377-9943","authenticated-orcid":false,"given":"Tim","family":"Nelson","sequence":"additional","affiliation":[{"name":"Brown University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0588-8649","authenticated-orcid":false,"given":"John","family":"Wrenn","sequence":"additional","affiliation":[{"name":"Brown University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5184-1975","authenticated-orcid":false,"given":"Shriram","family":"Krishnamurthi","sequence":"additional","affiliation":[{"name":"Brown University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,11,17]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1189136.1189182"},{"key":"e_1_3_2_1_2_1","volume-title":"Ante Up: A Framework to Strengthen Student-Based Testing of Assignments. In SIGCSE. 488\u2013493. https:\/\/doi.org\/10.1145\/2676723.2677247","author":"Bradshaw K.","year":"2015","unstructured":"Michael\u00a0 K. Bradshaw . 2015 . Ante Up: A Framework to Strengthen Student-Based Testing of Assignments. In SIGCSE. 488\u2013493. https:\/\/doi.org\/10.1145\/2676723.2677247 10.1145\/2676723.2677247 Michael\u00a0K. Bradshaw. 2015. Ante Up: A Framework to Strengthen Student-Based Testing of Assignments. In SIGCSE. 488\u2013493. https:\/\/doi.org\/10.1145\/2676723.2677247"},{"key":"e_1_3_2_1_3_1","volume-title":"Forge: A Tool and Language for Teaching Formal Methods. https:\/\/forge-fm.org Accessed 2022-09-21.","author":"Contributors Forge","year":"2022","unstructured":"Forge Contributors . 2022 . Forge: A Tool and Language for Teaching Formal Methods. https:\/\/forge-fm.org Accessed 2022-09-21. Forge Contributors. 2022. Forge: A Tool and Language for Teaching Formal Methods. https:\/\/forge-fm.org Accessed 2022-09-21."},{"key":"#cr-split#-e_1_3_2_1_4_1.1","doi-asserted-by":"crossref","unstructured":"Holger Danielsiek Wolfgang Paul and Jan Vahrenhold. 2012. Detecting and Understanding Students' Misconceptions Related to Algorithms and Data Structures. In SIGCSE. 21-26. https:\/\/doi.org\/10.1145\/2157136.2157148 10.1145\/2157136.2157148","DOI":"10.1145\/2157136.2157148"},{"key":"#cr-split#-e_1_3_2_1_4_1.2","doi-asserted-by":"crossref","unstructured":"Holger Danielsiek Wolfgang Paul and Jan Vahrenhold. 2012. Detecting and Understanding Students' Misconceptions Related to Algorithms and Data Structures. In SIGCSE. 21-26. https:\/\/doi.org\/10.1145\/2157136.2157148","DOI":"10.1145\/2157136.2157148"},{"key":"#cr-split#-e_1_3_2_1_5_1.1","doi-asserted-by":"crossref","unstructured":"Paul Denny John Hamer Andrew Luxton-Reilly and Helen\u00a0C. Purchase. 2008. PeerWise: Students Sharing their Multiple Choice Questions. In ICER. 51-58. https:\/\/doi.org\/10.1145\/1404520.1404526 10.1145\/1404520.1404526","DOI":"10.1145\/1404520.1404526"},{"key":"#cr-split#-e_1_3_2_1_5_1.2","doi-asserted-by":"crossref","unstructured":"Paul Denny John Hamer Andrew Luxton-Reilly and Helen\u00a0C. Purchase. 2008. PeerWise: Students Sharing their Multiple Choice Questions. In ICER. 51-58. https:\/\/doi.org\/10.1145\/1404520.1404526","DOI":"10.1145\/1404520.1404526"},{"key":"e_1_3_2_1_6_1","first-page":"1","article-title":"A Closer Look at Metacognitive Scaffolding: Solving Test Cases Before Programming","volume":"11","author":"Denny Paul","year":"2019","unstructured":"Paul Denny , James Prather , Brett\u00a0 A. Becker , Zachary Albrecht , Dastyni Loksa , and Raymond Pettit . 2019 . A Closer Look at Metacognitive Scaffolding: Solving Test Cases Before Programming . In Koli Calling. 11 : 1 \u2013 11 :10. https:\/\/doi.org\/10.1145\/3364510.3366170 10.1145\/3364510.3366170 Paul Denny, James Prather, Brett\u00a0A. Becker, Zachary Albrecht, Dastyni Loksa, and Raymond Pettit. 2019. A Closer Look at Metacognitive Scaffolding: Solving Test Cases Before Programming. In Koli Calling. 11:1\u201311:10. https:\/\/doi.org\/10.1145\/3364510.3366170","journal-title":"Koli Calling."},{"key":"e_1_3_2_1_7_1","article-title":"Improving Student Performance by Evaluating How Well Students Test Their Own Programs","volume":"3","author":"Edwards H.","year":"2003","unstructured":"Stephen\u00a0 H. Edwards . 2003 . Improving Student Performance by Evaluating How Well Students Test Their Own Programs . ACM Journal on Educational Resources in Computing 3 , 3 (2003), 1:1\u20131:24. https:\/\/doi.org\/10.1145\/1029994.1029995 10.1145\/1029994.1029995 Stephen\u00a0H. Edwards. 2003. Improving Student Performance by Evaluating How Well Students Test Their Own Programs. ACM Journal on Educational Resources in Computing 3, 3 (2003), 1:1\u20131:24. https:\/\/doi.org\/10.1145\/1029994.1029995","journal-title":"ACM Journal on Educational Resources in Computing"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"B. Glaser and A. Strauss. 1967. The Discovery of Grounded Theory: Strategies for Qualitative Research. Sociology Press.  B. Glaser and A. Strauss. 1967. The Discovery of Grounded Theory: Strategies for Qualitative Research. Sociology Press.","DOI":"10.1097\/00006199-196807000-00014"},{"key":"#cr-split#-e_1_3_2_1_9_1.1","doi-asserted-by":"crossref","unstructured":"Kenneth\u00a0J. Goldman Paul Gross Cinda Heeren Geoffrey\u00a0L. Herman Lisa\u00a0C. Kaczmarczyk Michael\u00a0C. Loui and Craig\u00a0B. Zilles. 2008. Identifying Important and Difficult Concepts in Introductory Computing Courses using a Delphi Process. In SIGCSE. 256-260. https:\/\/doi.org\/10.1145\/1352135.1352226 10.1145\/1352135.1352226","DOI":"10.1145\/1352322.1352226"},{"key":"#cr-split#-e_1_3_2_1_9_1.2","doi-asserted-by":"crossref","unstructured":"Kenneth\u00a0J. Goldman Paul Gross Cinda Heeren Geoffrey\u00a0L. Herman Lisa\u00a0C. Kaczmarczyk Michael\u00a0C. Loui and Craig\u00a0B. Zilles. 2008. Identifying Important and Difficult Concepts in Introductory Computing Courses using a Delphi Process. In SIGCSE. 256-260. https:\/\/doi.org\/10.1145\/1352135.1352226","DOI":"10.1145\/1352322.1352226"},{"key":"#cr-split#-e_1_3_2_1_10_1.1","doi-asserted-by":"crossref","unstructured":"Geoffrey\u00a0L. Herman Michael\u00a0C. Loui and Craig\u00a0B. Zilles. 2010. Creating the Digital Logic Concept Inventory. In SIGCSE. 102-106. https:\/\/doi.org\/10.1145\/1734263.1734298 10.1145\/1734263.1734298","DOI":"10.1145\/1734263.1734298"},{"key":"#cr-split#-e_1_3_2_1_10_1.2","doi-asserted-by":"crossref","unstructured":"Geoffrey\u00a0L. Herman Michael\u00a0C. Loui and Craig\u00a0B. Zilles. 2010. Creating the Digital Logic Concept Inventory. In SIGCSE. 102-106. https:\/\/doi.org\/10.1145\/1734263.1734298","DOI":"10.1145\/1734263.1734298"},{"key":"e_1_3_2_1_11_1","volume-title":"Software Abstractions: Logic, Language, and Analysis (2 ed.)","author":"Jackson Daniel","year":"2012","unstructured":"Daniel Jackson . 2012 . Software Abstractions: Logic, Language, and Analysis (2 ed.) . MIT Press . Daniel Jackson. 2012. Software Abstractions: Logic, Language, and Analysis (2 ed.). MIT Press."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338843"},{"key":"#cr-split#-e_1_3_2_1_14_1.1","doi-asserted-by":"crossref","unstructured":"Dastyni Loksa and Amy\u00a0J. Ko. 2016. The Role of Self-Regulation in Programming Problem Solving Process and Success. In ICER. 83-91. https:\/\/doi.org\/10.1145\/2960310.2960334 10.1145\/2960310.2960334","DOI":"10.1145\/2960310.2960334"},{"key":"#cr-split#-e_1_3_2_1_14_1.2","doi-asserted-by":"crossref","unstructured":"Dastyni Loksa and Amy\u00a0J. Ko. 2016. The Role of Self-Regulation in Programming Problem Solving Process and Success. In ICER. 83-91. https:\/\/doi.org\/10.1145\/2960310.2960334","DOI":"10.1145\/2960310.2960334"},{"key":"e_1_3_2_1_15_1","volume-title":"Testing First: Emphasizing Testing in Early Programming Courses. In SIGCSE. 4\u20138. https:\/\/doi.org\/10.1145\/1067445.1067451","author":"Marrero Will","year":"2005","unstructured":"Will Marrero and Amber Settle . 2005 . Testing First: Emphasizing Testing in Early Programming Courses. In SIGCSE. 4\u20138. https:\/\/doi.org\/10.1145\/1067445.1067451 10.1145\/1067445.1067451 Will Marrero and Amber Settle. 2005. Testing First: Emphasizing Testing in Early Programming Courses. In SIGCSE. 4\u20138. https:\/\/doi.org\/10.1145\/1067445.1067451"},{"key":"#cr-split#-e_1_3_2_1_16_1.1","doi-asserted-by":"crossref","unstructured":"James Prather Raymond Pettit Kayla\u00a0Holcomb McMurry Alani\u00a0L. Peters John Homer and Maxine\u00a0S. Cohen. 2018. Metacognitive Difficulties Faced by Novice Programmers in Automated Assessment Tools. In ICER. 41-50. https:\/\/doi.org\/10.1145\/3230977.3230981 10.1145\/3230977.3230981","DOI":"10.1145\/3230977.3230981"},{"key":"#cr-split#-e_1_3_2_1_16_1.2","doi-asserted-by":"crossref","unstructured":"James Prather Raymond Pettit Kayla\u00a0Holcomb McMurry Alani\u00a0L. Peters John Homer and Maxine\u00a0S. Cohen. 2018. Metacognitive Difficulties Faced by Novice Programmers in Automated Assessment Tools. In ICER. 41-50. https:\/\/doi.org\/10.1145\/3230977.3230981","DOI":"10.1145\/3230977.3230981"},{"key":"#cr-split#-e_1_3_2_1_17_1.1","doi-asserted-by":"crossref","unstructured":"Sam Saarinen Shriram Krishnamurthi Kathi Fisler and Preston Tunnell\u00a0Wilson. 2019. Harnessing the Wisdom of the Classes: Classsourcing and Machine Learning for Assessment Instrument Generation. In SIGCSE. 606-612. https:\/\/doi.org\/10.1145\/3287324.3287504 10.1145\/3287324.3287504","DOI":"10.1145\/3287324.3287504"},{"key":"#cr-split#-e_1_3_2_1_17_1.2","doi-asserted-by":"crossref","unstructured":"Sam Saarinen Shriram Krishnamurthi Kathi Fisler and Preston Tunnell\u00a0Wilson. 2019. Harnessing the Wisdom of the Classes: Classsourcing and Machine Learning for Assessment Instrument Generation. In SIGCSE. 606-612. https:\/\/doi.org\/10.1145\/3287324.3287504","DOI":"10.1145\/3287324.3287504"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/361219.361220"},{"key":"#cr-split#-e_1_3_2_1_19_1.1","doi-asserted-by":"crossref","unstructured":"Abigail Siegel Mia Santomauro Tristan Dyer Tim Nelson and Shriram Krishnamurthi. 2021. Prototyping Formal Methods Tools: A Protocol Analysis Case Study. In Protocols Strands and Logic - Essays Dedicated to Joshua Guttman on the Occasion of his 66.66th Birthday. 394-413. https:\/\/doi.org\/10.1007\/978-3-030-91631-2_22 10.1007\/978-3-030-91631-2_22","DOI":"10.1007\/978-3-030-91631-2_22"},{"key":"#cr-split#-e_1_3_2_1_19_1.2","doi-asserted-by":"crossref","unstructured":"Abigail Siegel Mia Santomauro Tristan Dyer Tim Nelson and Shriram Krishnamurthi. 2021. Prototyping Formal Methods Tools: A Protocol Analysis Case Study. In Protocols Strands and Logic - Essays Dedicated to Joshua Guttman on the Occasion of his 66.66th Birthday. 394-413. https:\/\/doi.org\/10.1007\/978-3-030-91631-2_22","DOI":"10.1007\/978-3-030-91631-2_22"},{"key":"#cr-split#-e_1_3_2_1_20_1.1","doi-asserted-by":"crossref","unstructured":"Allison\u00a0Elliott Tew and Mark Guzdial. 2010. Developing a Validated Assessment of Fundamental CS1 Concepts. In SIGCSE. 97-101. https:\/\/doi.org\/10.1145\/1734263.1734297 10.1145\/1734263.1734297","DOI":"10.1145\/1734263.1734297"},{"key":"#cr-split#-e_1_3_2_1_20_1.2","doi-asserted-by":"crossref","unstructured":"Allison\u00a0Elliott Tew and Mark Guzdial. 2010. Developing a Validated Assessment of Fundamental CS1 Concepts. In SIGCSE. 97-101. https:\/\/doi.org\/10.1145\/1734263.1734297","DOI":"10.1145\/1734263.1734297"},{"key":"e_1_3_2_1_21_1","volume-title":"Whalley and Nadia Kasto","author":"L.","year":"2014","unstructured":"Jacqueline\u00a0 L. Whalley and Nadia Kasto . 2014 . A Qualitative Think-Aloud Study of Novice Programmers\u2019 Code Writing Strategies. In ITiCSE. 279\u2013284. https:\/\/doi.org\/10.1145\/2591708.2591762 10.1145\/2591708.2591762 Jacqueline\u00a0L. Whalley and Nadia Kasto. 2014. A Qualitative Think-Aloud Study of Novice Programmers\u2019 Code Writing Strategies. In ITiCSE. 279\u2013284. https:\/\/doi.org\/10.1145\/2591708.2591762"},{"key":"#cr-split#-e_1_3_2_1_22_1.1","doi-asserted-by":"crossref","unstructured":"John Wrenn and Shriram Krishnamurthi. 2019. Executable Examples for Programming Problem Comprehension. In ICER. 131-139. https:\/\/doi.org\/10.1145\/3291279.3339416 10.1145\/3291279.3339416","DOI":"10.1145\/3291279.3339416"},{"key":"#cr-split#-e_1_3_2_1_22_1.2","doi-asserted-by":"crossref","unstructured":"John Wrenn and Shriram Krishnamurthi. 2019. Executable Examples for Programming Problem Comprehension. In ICER. 131-139. https:\/\/doi.org\/10.1145\/3291279.3339416","DOI":"10.1145\/3291279.3339416"},{"key":"#cr-split#-e_1_3_2_1_23_1.1","doi-asserted-by":"crossref","unstructured":"John Wrenn Shriram Krishnamurthi and Kathi Fisler. 2018. Who Tests the Testers?. In ICER. 51-59. https:\/\/doi.org\/10.1145\/3230977.3230999 10.1145\/3230977.3230999","DOI":"10.1145\/3230977.3230999"},{"key":"#cr-split#-e_1_3_2_1_23_1.2","doi-asserted-by":"crossref","unstructured":"John Wrenn Shriram Krishnamurthi and Kathi Fisler. 2018. Who Tests the Testers?. In ICER. 51-59. https:\/\/doi.org\/10.1145\/3230977.3230999","DOI":"10.1145\/3230977.3230999"},{"key":"e_1_3_2_1_24_1","unstructured":"John\u00a0Sinclair Wrenn. 2022. Executable Examples: Empowering Students to Hone Their Problem Comprehension. Ph.D. Dissertation. Brown University. https:\/\/repository.library.brown.edu\/studio\/item\/bdr:wgtu3pq6\/  John\u00a0Sinclair Wrenn. 2022. Executable Examples: Empowering Students to Hone Their Problem Comprehension. Ph.D. Dissertation. Brown University. https:\/\/repository.library.brown.edu\/studio\/item\/bdr:wgtu3pq6\/"}],"event":{"name":"Koli 2022: 22nd Koli Calling International Conference on Computing Education Research","location":"Koli Finland","acronym":"Koli 2022"},"container-title":["Proceedings of the 22nd Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3564721.3564726","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3564721.3564726","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:49:30Z","timestamp":1750182570000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3564721.3564726"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,17]]},"references-count":34,"alternative-id":["10.1145\/3564721.3564726","10.1145\/3564721"],"URL":"https:\/\/doi.org\/10.1145\/3564721.3564726","relation":{},"subject":[],"published":{"date-parts":[[2022,11,17]]},"assertion":[{"value":"2022-11-17","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}