{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,2]],"date-time":"2025-11-02T16:55:47Z","timestamp":1762102547639,"version":"3.40.3"},"publisher-location":"Cham","reference-count":48,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030587024"},{"type":"electronic","value":"9783030587031"}],"license":[{"start":{"date-parts":[[2020,9,9]],"date-time":"2020-09-09T00:00:00Z","timestamp":1599609600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,9,9]],"date-time":"2020-09-09T00:00:00Z","timestamp":1599609600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-58703-1_1","type":"book-chapter","created":{"date-parts":[[2020,9,8]],"date-time":"2020-09-08T06:02:47Z","timestamp":1599544967000},"page":"3-34","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Experiences and Lessons Learned Creating and Validating Concept Inventories for Cybersecurity"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1130-4678","authenticated-orcid":false,"given":"Alan T.","family":"Sherman","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9501-2295","authenticated-orcid":false,"given":"Geoffrey L.","family":"Herman","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0056-7819","authenticated-orcid":false,"given":"Linda","family":"Oliva","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3285-496X","authenticated-orcid":false,"given":"Peter A. H.","family":"Peterson","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0814-9956","authenticated-orcid":false,"given":"Enis","family":"Golaszewski","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6284-9972","authenticated-orcid":false,"given":"Seth","family":"Poulsen","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0821-1123","authenticated-orcid":false,"given":"Travis","family":"Scheponik","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2208-165X","authenticated-orcid":false,"given":"Akshita","family":"Gorti","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,9,9]]},"reference":[{"key":"1_CR1","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1177\/0146621603257518","volume":"27","author":"TM Bechger","year":"2003","unstructured":"Bechger, T.M., Maris, G., Verstralen, H., Beguin, A.A.: Using classical test theory in combination with item response theory. Appl. Psychol. Meas. 27, 319\u2013334 (2003)","journal-title":"Appl. Psychol. Meas."},{"key":"1_CR2","unstructured":"Brame, C.J.: Writing good multiple choice test questions 2019. https:\/\/cft.vanderbilt.edu\/guides-sub-pages\/writing-good-multiple-choice-test-questions\/. Accessed 19 Jan 2019"},{"key":"1_CR3","unstructured":"Brown, B.: Delphi process: a methodology used for the elicitation of opinions of experts. CA, USA, Rand Corporation, Santa Monica, September 1968"},{"key":"1_CR4","unstructured":"U.S. Department of\u00a0Labor Bureau\u00a0of Labor\u00a0Statistics. Information Security Analysts. Occupational Outlook Handbook, September 2019"},{"key":"1_CR5","unstructured":"George Washington University\u00a0Arlington Center. Cybersecurity education workshop, February 2014. https:\/\/research.gwu.edu\/sites\/g\/files\/zaxdzs2176\/f\/downloads\/CEW_FinalReport_040714.pdf"},{"key":"1_CR6","doi-asserted-by":"crossref","unstructured":"Chi, M.T.H.: Methods to assess the representations of experts\u2019 and novices\u2019 knowledge. In: Cambridge Handbook of Expertise and Expert Performance, pp. 167\u2013184 (2006)","DOI":"10.1017\/CBO9780511816796.010"},{"key":"1_CR7","volume-title":"CompTIA Security+ Guide to Network Security Fundamentals, Loose-Leaf Version","author":"M Ciampa","year":"2017","unstructured":"Ciampa, M.: CompTIA Security+ Guide to Network Security Fundamentals, Loose-Leaf Version, 6th edn. Course Technology Press, Boston (2017)","edition":"6"},{"key":"1_CR8","unstructured":"CompTIA. CASP (CAS-003) certification study guide: CompTIA IT certifications"},{"key":"1_CR9","unstructured":"International Information System Security\u00a0Certification Consortium: Certified information systems security professional. https:\/\/www.isc2.org\/cissp\/default.aspx. Accessed 14 Mar 2017"},{"key":"1_CR10","unstructured":"International Information Systems Security\u00a0Certification Consortium. GIAC certifications: The highest standard in cyber security certifications. https:\/\/www.giac.org\/"},{"key":"1_CR11","unstructured":"Cybersecurity and Infrastructure\u00a0Security Agency. The National Initiative for Cybersecurity Careers & Studies. https:\/\/niccs.us-cert.gov\/featured-stories\/take-cybersecurity-certification-prep-course"},{"issue":"8","key":"1_CR12","first-page":"1018","volume":"60","author":"J Epstein","year":"2013","unstructured":"Epstein, J.: The calculus concept inventory: measurement of the effect of teaching methodology in mathematics. Not. ACM 60(8), 1018\u20131025 (2013)","journal-title":"Not. ACM"},{"key":"1_CR13","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/5657.001.0001","volume-title":"Protocol Analysis","author":"KA Ericsson","year":"1993","unstructured":"Ericsson, K.A., Simon, H.A.: Protocol Analysis. MIT Press, Cambridge (1993)"},{"key":"1_CR14","doi-asserted-by":"crossref","unstructured":"Evans, D.L., Gray, G.L., Krause, S., Martin, J., Midkiff, C., Notaros, B.M., Pavelich, M., Rancour, D., Reed, T., Steif, P., Streveler, R., Wage, K.: Progress on concept inventory assessment tools. In 33rd Annual Frontiers in Education, 2003. FIE 2003, pp. T4G\u2013 1, December 2003","DOI":"10.1109\/FIE.2003.1263392"},{"key":"1_CR15","doi-asserted-by":"crossref","unstructured":"CSEC2017 Joint\u00a0Task Force. Cybersecurity curricula 2017. Technical report, CSEC2017 Joint task force, December 2017","DOI":"10.1145\/3422808"},{"issue":"3","key":"1_CR16","doi-asserted-by":"publisher","first-page":"38","DOI":"10.1109\/MC.2018.2882425","volume":"52","author":"D Gibson","year":"2019","unstructured":"Gibson, D., Anand, V., Dehlinger, J., Dierbach, C., Emmersen, T., Phillips, A.: Accredited undergraduate cybersecurity degrees: four approaches. Computer 52(3), 38\u201347 (2019)","journal-title":"Computer"},{"issue":"4","key":"1_CR17","doi-asserted-by":"publisher","first-page":"364","DOI":"10.1097\/00006199-196807000-00014","volume":"17","author":"BG Glaser","year":"1968","unstructured":"Glaser, B.G., Strauss, A.L., Strutzel, E.: The discovery of grounded theory: strategies for qualitative research. Nurs. Res. 17(4), 364 (1968)","journal-title":"Nurs. Res."},{"issue":"2","key":"1_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/1789934.1789935","volume":"10","author":"K Goldman","year":"2010","unstructured":"Goldman, K., Gross, P., Heeren, C., Herman, G.L., Kaczmarczy, L., Loui, M.C., Zilles, C.: Setting the scope of concept inventories for introductory computing subject. ACM Trans. Comput. Educ. 10(2), 1\u201329 (2010)","journal-title":"ACM Trans. Comput. Educ."},{"issue":"1","key":"1_CR19","doi-asserted-by":"publisher","first-page":"64","DOI":"10.1119\/1.18809","volume":"66","author":"R Hake","year":"1998","unstructured":"Hake, R.: Interactive-engagement versus traditional methods: a six-thousand-student survey of mechanics test data for introductory physics courses. Am. J. Phys. 66(1), 64\u201374 (1998)","journal-title":"Am. J. Phys."},{"key":"1_CR20","doi-asserted-by":"publisher","first-page":"07","DOI":"10.5751\/ES-00259-050107","volume":"5","author":"R Hake","year":"2001","unstructured":"Hake, R.: Lessons from the physics-education reform effort. Conserv. Ecol. 5, 07 (2001)","journal-title":"Conserv. Ecol."},{"key":"1_CR21","doi-asserted-by":"crossref","first-page":"253","DOI":"10.1111\/j.1745-3992.1993.tb00543.x","volume":"12","author":"RK Hambleton","year":"1993","unstructured":"Hambleton, R.K., Jones, R.J.: Comparison of classical test theory and item response theory and their applications to test development. Educ. Meas. Issues Pract. 12, 253\u2013262 (1993)","journal-title":"Educ. Meas. Issues Pract."},{"key":"1_CR22","doi-asserted-by":"crossref","unstructured":"Herman, G.L., Loui, M.C., Zilles, C.: Creating the digital logic concept inventory. In: Proceedings of the 41st ACM Technical Symposium on Computer Science Education, pp. 102\u2013106, January 2010","DOI":"10.1145\/1734263.1734298"},{"key":"1_CR23","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1119\/1.2343497","volume":"30","author":"D Hestenes","year":"1992","unstructured":"Hestenes, D., Wells, M., Swackhamer, G.: Force concept inventory. Phys. Teach. 30, 141\u2013166 (1992)","journal-title":"Phys. Teach."},{"key":"1_CR24","unstructured":"Jindeel, M.: I just don\u2019t get it: common security misconceptions. Master\u2019s thesis, University of Minnesota, June 2019"},{"key":"1_CR25","doi-asserted-by":"crossref","unstructured":"Association for computing machinery (ACM) joint task force on computing curricula and IEEE Computer Society. Computer Science Curricula: Curriculum Guidelines for Undergraduate Degree Programs in Computer Science. Association for Computing Machinery, New York (2013)","DOI":"10.1145\/2534860"},{"issue":"4","key":"1_CR26","doi-asserted-by":"publisher","first-page":"454","DOI":"10.1002\/jee.20104","volume":"104","author":"N Jorion","year":"2015","unstructured":"Jorion, N., Gane, B., James, K., Schroeder, L., DiBello, L., Pellegrino, J.: An analytic framework for evaluating the validity of concept inventory claims. J. Eng. Educ. 104(4), 454\u2013496 (2015)","journal-title":"J. Eng. Educ."},{"key":"1_CR27","doi-asserted-by":"crossref","unstructured":"Kittur, A., Chi, E.H., Suh, B.: Crowdsourcing user studies with Mechanical Turk. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 453\u2013456, April 2008","DOI":"10.1145\/1357054.1357127"},{"issue":"4","key":"1_CR28","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1002\/j.2168-9830.2010.tb01067.x","volume":"99","author":"TA Litzinger","year":"2010","unstructured":"Litzinger, T.A., Van Meter, P., Firetto, C.M., Passmore, L.J., Masters, C.B., Turns, S.R., Gray, G.L., Costanzo, F., Zappe, S.E.: A cognitive study of problem solving in statics. J. Eng. Educ. 99(4), 337\u2013353 (2010)","journal-title":"J. Eng. Educ."},{"issue":"3","key":"1_CR29","doi-asserted-by":"publisher","first-page":"303","DOI":"10.1002\/tea.3660300306","volume":"30","author":"JP Mestre","year":"1993","unstructured":"Mestre, J.P., Dufresne, R.J., Gerace, W.J., Hardiman, P.T., Touger, J.S.: Promoting skilled problem-solving behavior among beginning physics students. J. Res. Sci. Teach. 30(3), 303\u2013317 (1993)","journal-title":"J. Res. Sci. Teach."},{"key":"1_CR30","unstructured":"NIST. NICE framework. http:\/\/csrc.nist.gov\/nice\/framework\/. Accessed 8 Oct 2016"},{"key":"1_CR31","doi-asserted-by":"crossref","unstructured":"Offenberger, S., Herman, G.L., Peterson, P., Sherman, A.T., Golaszewski, E., Scheponik, T., Oliva, L.: Initial validation of the cybersecurity concept inventory: pilot testing and expert review. In: Proceedings of Frontiers in Education Conference, October 2019","DOI":"10.1109\/FIE43999.2019.9028652"},{"issue":"1","key":"1_CR32","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1002\/j.2168-9830.2005.tb00826.x","volume":"94","author":"BM Olds","year":"2005","unstructured":"Olds, B.M., Moskal, B.M., Miller, R.L.: Assessment in engineering education: evolution, approaches and future collaborations. J. Eng. Educ. 94(1), 13\u201325 (2005)","journal-title":"J. Eng. Educ."},{"issue":"11","key":"1_CR33","first-page":"11","volume":"61","author":"G Parekh","year":"2016","unstructured":"Parekh, G., DeLatte, D., Herman, G.L., Oliva, L., Phatak, D., Scheponik, T., Sherman, A.T.: Identifying core concepts of cybersecurity: results of two Delphi processes. IEEE Trans. Educ. 61(11), 11\u201320 (2016)","journal-title":"IEEE Trans. Educ."},{"key":"1_CR34","unstructured":"DiBello, L.V., James, K., Jorion, N. and Schroeder, L., Pellegrino, J.W.: Concept inventories as aids for instruction: a validity framework with examples of application. In: Proceedings of Research in Engineering Education Symposium, Madrid, Spain (2011)"},{"key":"1_CR35","volume-title":"Knowing What Students Know: The Science and Design of Educational Assessment","author":"J Pellegrino","year":"2001","unstructured":"Pellegrino, J., Chudowsky, N., Glaser, R.: Knowing What Students Know: The Science and Design of Educational Assessment. National Academy Press, Washington DC (2001)"},{"key":"1_CR36","unstructured":"Peterson, P.A.H, Jindeel, M., Straumann, A., Smith, J., Pederson, A., Geraci, B., Nowaczek, J., Powers, C., Kuutti, K.: The security misconceptions project, October 2019. https:\/\/secmisco.blogspot.com\/"},{"key":"1_CR37","doi-asserted-by":"crossref","unstructured":"Porter, L., Zingaro, D., Liao, S.N., Taylor, C., Webb, K.C., Lee, C., Clancy, M.: BDSI: a validated concept inventory for basic data structures. In: Proceedings of the 2019 ACM Conference on International Computing Education Research, ICER 2019, pp. 111\u2013119. Association for Computing Machinery, New York (2019)","DOI":"10.1145\/3291279.3339404"},{"key":"1_CR38","unstructured":"Scheponik, T., Golaszewski, E., Herman, G., Offenberger, S., Oliva, L., Peterson, P.A.H., Sherman, A.T.: Investigating crowdsourcing to generate distractors for multiple-choice assessments (2019). https:\/\/arxiv.org\/pdf\/1909.04230.pdf"},{"key":"1_CR39","doi-asserted-by":"crossref","unstructured":"Scheponik, T., Golaszewski, E., Herman, G., Offenberger, S., Oliva, L., Peterson, P.A. and Sherman, A.T.: Investigating crowdsourcing to generate distractors for multiple-choice assessments. In: Choo, K.K.R., Morris, T.H., Peterson, G.L. (eds.) National Cyber Summit (NCS) Research Track, pp. 185\u2013201. Springer, Cham (2020)","DOI":"10.1007\/978-3-030-31239-8_15"},{"key":"1_CR40","doi-asserted-by":"crossref","unstructured":"Scheponik, T., Sherman, A.T., DeLatte, D., Phatak, D., Oliva, L., Thompson, J. and Herman, G.L.: How students reason about cybersecurity concepts. In: IEEE Frontiers in Education Conference (FIE), pp. 1\u20135, October 2016","DOI":"10.1109\/FIE.2016.7757363"},{"key":"1_CR41","unstructured":"Offensive Security. Penetration Testing with Kali Linux (PWK). https:\/\/www.offensive-security.com\/pwk-oscp\/"},{"issue":"4","key":"1_CR42","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1080\/01611194.2017.1362063","volume":"42","author":"AT Sherman","year":"2018","unstructured":"Sherman, A.T., DeLatte, D., Herman, G.L., Neary, M., Oliva, L., Phatak, D., Scheponik, T., Thompson, J.: Cybersecurity: exploring core concepts through six scenarios. Cryptologia 42(4), 337\u2013377 (2018)","journal-title":"Cryptologia"},{"issue":"6","key":"1_CR43","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1109\/MSEC.2019.2929812","volume":"17","author":"AT Sherman","year":"2019","unstructured":"Sherman, A.T., Oliva, L., Golaszewski, E., Phatak, D., Scheponik, T., Herman, G.L., Choi, D.S., Offenberger, S.E., Peterson, P., Dykstra, J., Bard, G.V., Chattopadhyay, A., Sharevski, F., Verma, R., Vrecenar, R.: The CATS hackathon: creating and refining test items for cybersecurity concept inventories. IEEE Secur. Priv. 17(6), 77\u201383 (2019)","journal-title":"IEEE Secur. Priv."},{"key":"1_CR44","unstructured":"Sherman, A.T., Oliva, L., DeLatte, D., Golaszewski, E., Neary, M., Patsourakos, K., Phatak, D., Scheponik, T., Herman, G.L. and Thompson, J.: Creating a cybersecurity concept inventory: a status report on the CATS Project. In: 2017 National Cyber Summit, June 2017"},{"issue":"1","key":"1_CR45","first-page":"5","volume":"2018","author":"JD Thompson","year":"2018","unstructured":"Thompson, J.D., Herman, G.L., Scheponik, T., Oliva, L., Sherman, A.T., Golaszewski, E., Phatak, D., Patsourakos, K.: Student misconceptions about cybersecurity concepts: analysis of think-aloud interviews. J. Cybersecurity Educ. Res. Pract. 2018(1), 5 (2018)","journal-title":"J. Cybersecurity Educ. Res. Pract."},{"key":"1_CR46","volume-title":"CEH Certified Ethical Hacker All-in-One Exam Guide","author":"M Walker","year":"2011","unstructured":"Walker, M.: CEH Certified Ethical Hacker All-in-One Exam Guide, 1st edn. McGraw-Hill Osborne Media, USA (2011)","edition":"1"},{"issue":"1","key":"1_CR47","doi-asserted-by":"publisher","first-page":"010116","DOI":"10.3847\/AER2010024","volume":"9","author":"CS Wallace","year":"2010","unstructured":"Wallace, C.S., Bailey, J.M.: Do concept inventories actually measure anything? Astron. Educ. Rev. 9(1), 010116 (2010)","journal-title":"Astron. Educ. Rev."},{"key":"1_CR48","unstructured":"West, M., Herman, G.L. and Zilles, C.: PrairieLearn: mastery-based online problem solving with adaptive scoring and recommendations driven by machine learning. In: 2015 ASEE Annual Conference and Exposition, ASEE Conferences, Seattle, Washington, June 2015"}],"container-title":["Advances in Intelligent Systems and Computing","National Cyber Summit (NCS) Research Track 2020"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-58703-1_1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,8,13]],"date-time":"2024-08-13T11:24:36Z","timestamp":1723548276000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-58703-1_1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,9,9]]},"ISBN":["9783030587024","9783030587031"],"references-count":48,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-58703-1_1","relation":{},"ISSN":["2194-5357","2194-5365"],"issn-type":[{"type":"print","value":"2194-5357"},{"type":"electronic","value":"2194-5365"}],"subject":[],"published":{"date-parts":[[2020,9,9]]},"assertion":[{"value":"9 September 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"NCS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"National Cyber Summit","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Huntsville, AL","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"USA","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 June 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"4 June 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ncs2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.nationalcybersummit.com\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}