{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,28]],"date-time":"2025-07-28T21:45:07Z","timestamp":1753739107652,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,11,18]],"date-time":"2021-11-18T00:00:00Z","timestamp":1637193600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,11,18]]},"DOI":"10.1145\/3488042.3488045","type":"proceedings-article","created":{"date-parts":[[2021,11,10]],"date-time":"2021-11-10T23:28:49Z","timestamp":1636586929000},"page":"1-11","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Wrong Answers for Wrong Reasons: The\u00a0Risks\u00a0of\u00a0Ad\u00a0Hoc\u00a0Instruments"],"prefix":"10.1145","author":[{"given":"Luca","family":"Chiodini","sequence":"first","affiliation":[{"name":"Software Institute - Universit\u00e0 della Svizzera italiana, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthias","family":"Hauswirth","sequence":"additional","affiliation":[{"name":"Software Institute - Universit\u00e0 della Svizzera italiana, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2021,11,18]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844601"},{"volume-title":"Concept Inventories in Computer Science for the Topic Discrete Mathematics","author":"Almstrum L.","key":"e_1_3_2_2_2_1","unstructured":"Vicki\u00a0 L. Almstrum , Peter\u00a0 B. Henderson , Valerie Harvey , Cinda Heeren , William Marion , Charles Riedesel , Leen-Kiat Soh , and Allison\u00a0Elliott Tew . 2006. Concept Inventories in Computer Science for the Topic Discrete Mathematics . In Working Group Reports on ITiCSE on Innovation and Technology in Computer Science Education(ITiCSE-WGR \u201906). ACM, New York, NY, USA, 132\u2013145. https:\/\/doi.org\/10.1145\/1189215.1189182 10.1145\/1189215.1189182 Vicki\u00a0L. Almstrum, Peter\u00a0B. Henderson, Valerie Harvey, Cinda Heeren, William Marion, Charles Riedesel, Leen-Kiat Soh, and Allison\u00a0Elliott Tew. 2006. Concept Inventories in Computer Science for the Topic Discrete Mathematics. In Working Group Reports on ITiCSE on Innovation and Technology in Computer Science Education(ITiCSE-WGR \u201906). ACM, New York, NY, USA, 132\u2013145. https:\/\/doi.org\/10.1145\/1189215.1189182"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387386"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428029.3428039"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3059009.3059024"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3430665.3456343"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287324.3287393"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2839509.2844567"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339427"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387393"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3231000"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3141880.3141895"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3304221.3319747"},{"key":"#cr-split#-e_1_3_2_2_14_1.1","doi-asserted-by":"crossref","unstructured":"Sally Fincher and Marian Petre (Eds.). 2005. Computer Science Education Research. Taylor & Francis. 95-110 pages. https:\/\/doi.org\/10.1201\/9781482287325-18 10.1201\/9781482287325-18","DOI":"10.1201\/9781482287325"},{"key":"#cr-split#-e_1_3_2_2_14_1.2","doi-asserted-by":"crossref","unstructured":"Sally Fincher and Marian Petre (Eds.). 2005. Computer Science Education Research. Taylor & Francis. 95-110 pages. https:\/\/doi.org\/10.1201\/9781482287325-18","DOI":"10.1201\/9781482287325"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352135.1352226"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3231002"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3059009.3059032"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387403"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3279720.3279726"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3279720.3279728"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387392"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2899415.2899427"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960328"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/359369.359393"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106168"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1080\/08993408.2018.1562145"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3372782.3406264"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339420"},{"volume-title":"Proceedings of the 2017 ACM Conference on International Computing Education Research. ACM, Tacoma Washington USA, 93\u2013100","author":"A.","key":"e_1_3_2_2_29_1","unstructured":"Michael\u00a0 A. Miljanovic and Jeremy\u00a0S. Bradbury. 2017. RoboBUG: A Serious Game for Learning Debugging Techniques . In Proceedings of the 2017 ACM Conference on International Computing Education Research. ACM, Tacoma Washington USA, 93\u2013100 . https:\/\/doi.org\/10.1145\/3105726.3106173 10.1145\/3105726.3106173 Michael\u00a0A. Miljanovic and Jeremy\u00a0S. Bradbury. 2017. RoboBUG: A Serious Game for Learning Debugging Techniques. In Proceedings of the 2017 ACM Conference on International Computing Education Research. ACM, Tacoma Washington USA, 93\u2013100. https:\/\/doi.org\/10.1145\/3105726.3106173"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106191"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960330"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106178"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960316"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2899415.2925503"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339404"},{"key":"e_1_3_2_2_36_1","first-page":"5","article-title":"Development and Validation of the Middle Grades Computer Science Concept Inventory (MG-CSCI) Assessment","volume":"16","author":"Rachmatullah Arif","year":"2020","unstructured":"Arif Rachmatullah , Bita Akram , Danielle Boulden , Bradford Mott , Kristy Boyer , James Lester , and Eric Wiebe . 2020 . Development and Validation of the Middle Grades Computer Science Concept Inventory (MG-CSCI) Assessment . EURASIA Journal of Mathematics, Science and Technology Education 16 , 5 (Feb. 2020). https:\/\/doi.org\/10.29333\/ejmste\/116600 10.29333\/ejmste Arif Rachmatullah, Bita Akram, Danielle Boulden, Bradford Mott, Kristy Boyer, James Lester, and Eric Wiebe. 2020. Development and Validation of the Middle Grades Computer Science Concept Inventory (MG-CSCI) Assessment. EURASIA Journal of Mathematics, Science and Technology Education 16, 5 (Feb. 2020). https:\/\/doi.org\/10.29333\/ejmste\/116600","journal-title":"EURASIA Journal of Mathematics, Science and Technology Education"},{"key":"e_1_3_2_2_37_1","volume-title":"Proceedings of the 19th Koli Calling International Conference on Computing Education Research. ACM, Koli Finland, 1\u201310","author":"Gerald\u00a0Soosai Raj Adalbert","year":"2019","unstructured":"Adalbert Gerald\u00a0Soosai Raj , Hanqi Zhang , Viren Abhyankar , Saswati Mukerjee , Eda Zhang , Jim Williams , Richard Halverson , and Jignesh\u00a0 M. Patel . 2019 . Impact of Bilingual CS Education on Student Learning and Engagement in a Data Structures Course . In Proceedings of the 19th Koli Calling International Conference on Computing Education Research. ACM, Koli Finland, 1\u201310 . https:\/\/doi.org\/10.1145\/3364510.3364518 10.1145\/3364510.3364518 Adalbert Gerald\u00a0Soosai Raj, Hanqi Zhang, Viren Abhyankar, Saswati Mukerjee, Eda Zhang, Jim Williams, Richard Halverson, and Jignesh\u00a0M. Patel. 2019. Impact of Bilingual CS Education on Student Learning and Engagement in a Data Structures Course. In Proceedings of the 19th Koli Calling International Conference on Computing Education Research. ACM, Koli Finland, 1\u201310. https:\/\/doi.org\/10.1145\/3364510.3364518"},{"key":"e_1_3_2_2_38_1","volume-title":"Recruiting Participants for Large-Scale Random Assignment Experiments in School Settings. Grantee Submission","author":"Roschelle Jeremy","year":"2014","unstructured":"Jeremy Roschelle , Mingyu Feng , H.\u00a0 Alix Gallagher , Robert Murphy , Christopher Harris , Danae Kamdar , and Gucci Trinidad . 2014. Recruiting Participants for Large-Scale Random Assignment Experiments in School Settings. Grantee Submission ( 2014 ). Jeremy Roschelle, Mingyu Feng, H.\u00a0Alix Gallagher, Robert Murphy, Christopher Harris, Danae Kamdar, and Gucci Trinidad. 2014. Recruiting Participants for Large-Scale Random Assignment Experiments in School Settings. Grantee Submission (2014)."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339408"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3059009.3059037"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611417632"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387390"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3197091.3197097"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3304221.3319787"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3197091.3197102"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428029.3428033"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/1028174.971391"},{"key":"e_1_3_2_2_48_1","first-page":"12","article-title":"More Remarks on Multiple Choice Questions","volume":"52","author":"Varney N.","year":"1998","unstructured":"Robert\u00a0 N. Varney . 1998 . More Remarks on Multiple Choice Questions . American Journal of Physics 52 , 12 (June 1998), 1069. https:\/\/doi.org\/10.1119\/1.13762 10.1119\/1.13762 Robert\u00a0N. Varney. 1998. More Remarks on Multiple Choice Questions. American Journal of Physics 52, 12 (June 1998), 1069. https:\/\/doi.org\/10.1119\/1.13762","journal-title":"American Journal of Physics"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3304221.3319760"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3059009.3059031"},{"key":"e_1_3_2_2_51_1","volume-title":"Proceedings of the 7th Australasian Conference on Computing Education -","volume":"116","author":"Woodford Karyn","year":"2005","unstructured":"Karyn Woodford and Peter Bancroft . 2005 . Multiple Choice Questions Not Considered Harmful . In Proceedings of the 7th Australasian Conference on Computing Education - Volume 42(ACE \u201905). Australian Computer Society, Inc., AUS, 109\u2013 116 . Karyn Woodford and Peter Bancroft. 2005. Multiple Choice Questions Not Considered Harmful. In Proceedings of the 7th Australasian Conference on Computing Education - Volume 42(ACE \u201905). Australian Computer Society, Inc., AUS, 109\u2013116."},{"key":"e_1_3_2_2_52_1","volume-title":"Sowing the Seeds: A Landscape Study on Assessment in Secondary Computer Science Education. (July","author":"Yadav Aman","year":"2015","unstructured":"Aman Yadav , David Burkhart , Daniel Moix , Eric Snow , Padmaja Bandaru , and Lissa Clayborn . 2015. Sowing the Seeds: A Landscape Study on Assessment in Secondary Computer Science Education. (July 2015 ). Aman Yadav, David Burkhart, Daniel Moix, Eric Snow, Padmaja Bandaru, and Lissa Clayborn. 2015. Sowing the Seeds: A Landscape Study on Assessment in Secondary Computer Science Education. (July 2015)."},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428029.3428050"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291279.3339419"}],"event":{"name":"Koli Calling '21: 21st Koli Calling International Conference on Computing Education Research","acronym":"Koli Calling '21","location":"Joensuu Finland"},"container-title":["Proceedings of the 21st Koli Calling International Conference on Computing Education Research"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3488042.3488045","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3488042.3488045","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:24:32Z","timestamp":1750195472000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3488042.3488045"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,11,18]]},"references-count":55,"alternative-id":["10.1145\/3488042.3488045","10.1145\/3488042"],"URL":"https:\/\/doi.org\/10.1145\/3488042.3488045","relation":{},"subject":[],"published":{"date-parts":[[2021,11,18]]},"assertion":[{"value":"2021-11-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}