{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,31]],"date-time":"2025-12-31T11:55:05Z","timestamp":1767182105224,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,8,3]],"date-time":"2022-08-03T00:00:00Z","timestamp":1659484800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-sa\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000138","name":"U.S. Department of Education","doi-asserted-by":"publisher","award":["U411C190092"],"award-info":[{"award-number":["U411C190092"]}],"id":[{"id":"10.13039\/100000138","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1923136, 1660871"],"award-info":[{"award-number":["1923136, 1660871"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,8,3]]},"DOI":"10.1145\/3501385.3543979","type":"proceedings-article","created":{"date-parts":[[2022,8,4]],"date-time":"2022-08-04T04:08:58Z","timestamp":1659586138000},"page":"2-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":13,"title":["A Pair of ACES: An Analysis of Isomorphic Questions on an Elementary Computing Assessment"],"prefix":"10.1145","author":[{"given":"Miranda C.","family":"Parker","sequence":"first","affiliation":[{"name":"University of California, Irvine, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Leiny","family":"Garcia","sequence":"additional","affiliation":[{"name":"University of California, Irvine, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yvonne S.","family":"Kao","sequence":"additional","affiliation":[{"name":"WestEd, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Diana","family":"Franklin","sequence":"additional","affiliation":[{"name":"University of Chicago, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Susan","family":"Krause","sequence":"additional","affiliation":[{"name":"University of Chicago, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mark","family":"Warschauer","sequence":"additional","affiliation":[{"name":"University of California, Irvine, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,8,3]]},"reference":[{"volume-title":"Standards for educational and psychological testing","author":"American Educational\u00a0Research Association 2018.","key":"e_1_3_2_1_1_1","unstructured":"American Educational\u00a0Research Association 2018. Standards for educational and psychological testing . American Educational Research Association . American Educational\u00a0Research Association 2018. Standards for educational and psychological testing. American Educational Research Association."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328778.3366849"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40692-019-00147-3"},{"volume-title":"Coding as a playground: Programming and computational thinking in the early childhood classroom","author":"Bers Marina\u00a0Umaschi","key":"e_1_3_2_1_4_1","unstructured":"Marina\u00a0Umaschi Bers . 2020. Coding as a playground: Programming and computational thinking in the early childhood classroom . Routledge . Marina\u00a0Umaschi Bers. 2020. Coding as a playground: Programming and computational thinking in the early childhood classroom. Routledge."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1177\/1094428107301102"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328778.3366829"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 2012 annual meeting of the American educational research association","author":"Brennan Karen","year":"2012","unstructured":"Karen Brennan and Mitchel Resnick . 2012 . New frameworks for studying and assessing the development of computational thinking . In Proceedings of the 2012 annual meeting of the American educational research association , Vancouver, Canada, Vol.\u00a01. 25. Karen Brennan and Mitchel Resnick. 2012. New frameworks for studying and assessing the development of computational thinking. In Proceedings of the 2012 annual meeting of the American educational research association, Vancouver, Canada, Vol.\u00a01. 25."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/CSEET49119.2020.9206195"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2017.03.001"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1744-6570.1998.tb00722.x"},{"key":"e_1_3_2_1_11_1","volume-title":"Coefficient alpha and the internal structure of tests. psychometrika 16, 3","author":"Cronbach J","year":"1951","unstructured":"Lee\u00a0 J Cronbach . 1951. Coefficient alpha and the internal structure of tests. psychometrika 16, 3 ( 1951 ), 297\u2013334. Lee\u00a0J Cronbach. 1951. Coefficient alpha and the internal structure of tests. psychometrika 16, 3 (1951), 297\u2013334."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46747-4_3"},{"key":"e_1_3_2_1_13_1","volume-title":"Bebras\u2013A Sustainable Community Building Model for the Concept Based Learning of Informatics and Computational Thinking.Informatics in education 15, 1","author":"Dagiene Valentina","year":"2016","unstructured":"Valentina Dagiene and Gabriele Stupuriene . 2016. Bebras\u2013A Sustainable Community Building Model for the Concept Based Learning of Informatics and Computational Thinking.Informatics in education 15, 1 ( 2016 ), 25\u201344. Valentina Dagiene and Gabriele Stupuriene. 2016. Bebras\u2013A Sustainable Community Building Model for the Concept Based Learning of Informatics and Computational Thinking.Informatics in education 15, 1 (2016), 25\u201344."},{"key":"e_1_3_2_1_14_1","unstructured":"Laura\u00a0E de Ruiter and Marina\u00a0U Bers. 2021. The Coding Stages Assessment: development and validation of an instrument for assessing young children\u2019s proficiency in the ScratchJr programming language. Computer Science Education(2021) 1\u201330.  Laura\u00a0E de Ruiter and Marina\u00a0U Bers. 2021. The Coding Stages Assessment: development and validation of an instrument for assessing young children\u2019s proficiency in the ScratchJr programming language. Computer Science Education(2021) 1\u201330."},{"key":"e_1_3_2_1_15_1","unstructured":"[\n  15\n  ]  International\u00a0Society for Technology\u00a0in Education and Computer Science\u00a0Teachers Association.[n.d.]. https:\/\/id.iste.org\/docs\/ct-documents\/computational-thinking-operational-definition-flyer.pdf  [15] International\u00a0Society for Technology\u00a0in Education and Computer Science\u00a0Teachers Association.[n.d.]. https:\/\/id.iste.org\/docs\/ct-documents\/computational-thinking-operational-definition-flyer.pdf"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432413"},{"key":"e_1_3_2_1_17_1","volume-title":"Analogical problem solving. Cognitive psychology 12, 3","author":"Gick L","year":"1980","unstructured":"Mary\u00a0 L Gick and Keith\u00a0 J Holyoak . 1980. Analogical problem solving. Cognitive psychology 12, 3 ( 1980 ), 306\u2013355. Mary\u00a0L Gick and Keith\u00a0J Holyoak. 1980. Analogical problem solving. Cognitive psychology 12, 3 (1980), 306\u2013355."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.cognition.2018.02.013"},{"key":"e_1_3_2_1_19_1","unstructured":"John\u00a0R Hayes and Herbert\u00a0A Simon. 1977. Psychological differences among problem isomorphs. Cognitive theory 2(1977) 21\u201341.  John\u00a0R Hayes and Herbert\u00a0A Simon. 1977. Psychological differences among problem isomorphs. Cognitive theory 2(1977) 21\u201341."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1080\/03640210701863933"},{"key":"e_1_3_2_1_21_1","volume-title":"Why are some problems hard? Evidence from Tower of Hanoi. Cognitive psychology 17, 2","author":"Kotovsky Kenneth","year":"1985","unstructured":"Kenneth Kotovsky , John\u00a0 R Hayes , and Herbert\u00a0 A Simon . 1985. Why are some problems hard? Evidence from Tower of Hanoi. Cognitive psychology 17, 2 ( 1985 ), 248\u2013294. Kenneth Kotovsky, John\u00a0R Hayes, and Herbert\u00a0A Simon. 1985. Why are some problems hard? Evidence from Tower of Hanoi. Cognitive psychology 17, 2 (1985), 248\u2013294."},{"key":"e_1_3_2_1_22_1","unstructured":"Patrick\u00a0C Kyllonen. 2002. Item generation for repeated testing of human performance. Item generation for test development(2002) 251\u2013276.  Patrick\u00a0C Kyllonen. 2002. Item generation for repeated testing of human performance. Item generation for test development(2002) 251\u2013276."},{"key":"e_1_3_2_1_23_1","volume-title":"Situational judgment tests in high-stakes settings: Issues and strategies with generating alternate forms.Journal of Applied Psychology 92, 4","author":"Lievens Filip","year":"2007","unstructured":"Filip Lievens and Paul\u00a0 R Sackett . 2007. Situational judgment tests in high-stakes settings: Issues and strategies with generating alternate forms.Journal of Applied Psychology 92, 4 ( 2007 ), 1043. Filip Lievens and Paul\u00a0R Sackett. 2007. Situational judgment tests in high-stakes settings: Issues and strategies with generating alternate forms.Journal of Applied Psychology 92, 4 (2007), 1043."},{"key":"e_1_3_2_1_24_1","unstructured":"James Lockwood and Aidan Mooney. 2017. Computational thinking in education: Where does it fit? A systematic literary review. arXiv preprint arXiv:1703.07659(2017).  James Lockwood and Aidan Mooney. 2017. Computational thinking in education: Where does it fit? A systematic literary review. arXiv preprint arXiv:1703.07659(2017)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3494579"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3381891"},{"key":"e_1_3_2_1_27_1","volume-title":"Should you show me the money? Concrete objects both hurt and help performance on mathematics problems. Learning and instruction 19, 2","author":"McNeil M","year":"2009","unstructured":"Nicole\u00a0 M McNeil , David\u00a0 H Uttal , Linda Jarvin , and Robert\u00a0 J Sternberg . 2009. Should you show me the money? Concrete objects both hurt and help performance on mathematics problems. Learning and instruction 19, 2 ( 2009 ), 171\u2013184. Nicole\u00a0M McNeil, David\u00a0H Uttal, Linda Jarvin, and Robert\u00a0J Sternberg. 2009. Should you show me the money? Concrete objects both hurt and help performance on mathematics problems. Learning and instruction 19, 2 (2009), 171\u2013184."},{"volume-title":"Psychometric Theory","author":"Nunnally Jum","key":"e_1_3_2_1_28_1","unstructured":"Jum Nunnally and Ira\u00a0 H Bernstein . 1994. Psychometric Theory . The McGraw-Hill Companies . Jum Nunnally and Ira\u00a0H Bernstein. 1994. Psychometric Theory. The McGraw-Hill Companies."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2960310.2960316"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3408877.3432376"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3230977.3230987"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341525.3387413"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2016911.2016923"},{"key":"e_1_3_2_1_34_1","unstructured":"Project Quantum. [n.d.]. Project Quantum \u2013 A Collection of Computing Quizzes. https:\/\/diagnosticquestions.com\/quantum  Project Quantum. [n.d.]. Project Quantum \u2013 A Collection of Computing Quizzes. https:\/\/diagnosticquestions.com\/quantum"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10956-020-09831-x"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3105726.3106166"},{"key":"e_1_3_2_1_37_1","volume-title":"Which cognitive abilities underlie computational thinking? Criterion validity of the Computational Thinking Test. Computers in human behavior 72","author":"Rom\u00e1n-Gonz\u00e1lez Marcos","year":"2017","unstructured":"Marcos Rom\u00e1n-Gonz\u00e1lez , Juan-Carlos P\u00e9rez-Gonz\u00e1lez , and Carmen Jim\u00e9nez-Fern\u00e1ndez . 2017. Which cognitive abilities underlie computational thinking? Criterion validity of the Computational Thinking Test. Computers in human behavior 72 ( 2017 ), 678\u2013691. Marcos Rom\u00e1n-Gonz\u00e1lez, Juan-Carlos P\u00e9rez-Gonz\u00e1lez, and Carmen Jim\u00e9nez-Fern\u00e1ndez. 2017. Which cognitive abilities underlie computational thinking? Criterion validity of the Computational Thinking Test. Computers in human behavior 72 (2017), 678\u2013691."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3430665.3456355"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.edurev.2017.09.003"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2019.103798"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328778.3366903"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1177\/0735633117743918"},{"key":"e_1_3_2_1_43_1","volume-title":"Examining the impact of question surface features on students","author":"Weston Michele","year":"2015","unstructured":"Michele Weston , Kevin\u00a0 C Haudek , Luanna Prevost , Mark Urban-Lurain , and John Merrill . 2015. Examining the impact of question surface features on students \u2019 answers to constructed-response questions on photosynthesis. CBE\u2014Life Sciences Education 14, 2 ( 2015 ), ar19. Michele Weston, Kevin\u00a0C Haudek, Luanna Prevost, Mark Urban-Lurain, and John Merrill. 2015. Examining the impact of question surface features on students\u2019 answers to constructed-response questions on photosynthesis. CBE\u2014Life Sciences Education 14, 2 (2015), ar19."},{"volume-title":"thought, and reality: Selected writings of Benjamin Lee Whorf","author":"Whorf Benjamin\u00a0Lee","key":"e_1_3_2_1_44_1","unstructured":"Benjamin\u00a0Lee Whorf . 2012. Language , thought, and reality: Selected writings of Benjamin Lee Whorf . MIT press . Benjamin\u00a0Lee Whorf. 2012. Language, thought, and reality: Selected writings of Benjamin Lee Whorf. MIT press."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1118178.1118215"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/EDUCON45650.2020.9125368"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1163\/1568537054068660"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676723.2677239"}],"event":{"name":"ICER 2022: ACM Conference on International Computing Education Research","sponsor":["SIGCSE ACM Special Interest Group on Computer Science Education"],"location":"Lugano and Virtual Event Switzerland","acronym":"ICER 2022"},"container-title":["Proceedings of the 2022 ACM Conference on International Computing Education Research - Volume 1"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3501385.3543979","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3501385.3543979","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:20Z","timestamp":1750188620000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3501385.3543979"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,3]]},"references-count":48,"alternative-id":["10.1145\/3501385.3543979","10.1145\/3501385"],"URL":"https:\/\/doi.org\/10.1145\/3501385.3543979","relation":{},"subject":[],"published":{"date-parts":[[2022,8,3]]},"assertion":[{"value":"2022-08-03","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}