{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T21:17:22Z","timestamp":1742937442596,"version":"3.40.3"},"publisher-location":"Cham","reference-count":31,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783031330223"},{"type":"electronic","value":"9783031330230"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-33023-0_11","type":"book-chapter","created":{"date-parts":[[2023,5,25]],"date-time":"2023-05-25T07:02:38Z","timestamp":1684998158000},"page":"122-134","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Automated Assessment in\u00a0Computer Science: A Bibliometric Analysis of\u00a0the\u00a0Literature"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0394-0527","authenticated-orcid":false,"given":"Jos\u00e9 Carlos","family":"Paiva","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0507-7504","authenticated-orcid":false,"given":"\u00c1lvaro","family":"Figueira","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8409-0300","authenticated-orcid":false,"given":"Jos\u00e9 Paulo","family":"Leal","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,5,26]]},"reference":[{"issue":"2","key":"11_CR1","doi-asserted-by":"publisher","first-page":"83","DOI":"10.1080\/08993400500150747","volume":"15","author":"KM Ala-Mutka","year":"2005","unstructured":"Ala-Mutka, K.M.: A survey of automated assessment approaches for programming assignments. Comput. Sci. Educ. 15(2), 83\u2013102 (2005). https:\/\/doi.org\/10.1080\/08993400500150747","journal-title":"Comput. Sci. Educ."},{"key":"11_CR2","doi-asserted-by":"publisher","DOI":"10.1533\/9781780630182","volume-title":"Measuring Academic Research","author":"A Andr\u00e9s","year":"2009","unstructured":"Andr\u00e9s, A.: Measuring Academic Research. Chandos Publishing (Oxford), Witney, England (2009)"},{"issue":"4","key":"11_CR3","doi-asserted-by":"publisher","first-page":"959","DOI":"10.1016\/j.joi.2017.08.007","volume":"11","author":"M Aria","year":"2017","unstructured":"Aria, M., Cuccurullo, C.: bibliometrix: an r-tool for comprehensive science mapping analysis. J. Informetrics 11(4), 959\u2013975 (2017). https:\/\/doi.org\/10.1016\/j.joi.2017.08.007","journal-title":"J. Informetrics"},{"issue":"4","key":"11_CR4","doi-asserted-by":"publisher","first-page":"561","DOI":"10.1080\/10508406.2014.954750","volume":"23","author":"P Blikstein","year":"2014","unstructured":"Blikstein, P., Worsley, M., Piech, C., Sahami, M., Cooper, S., Koller, D.: Programming pluralism: using learning analytics to detect patterns in the learning of computer programming. J. Learn. Sci. 23(4), 561\u2013599 (2014). https:\/\/doi.org\/10.1080\/10508406.2014.954750","journal-title":"J. Learn. Sci."},{"key":"11_CR5","unstructured":"Clarivate: Web of science core collection (2022). https:\/\/www.webofscience.com\/wos\/woscc\/summary\/f75398e2-c55c-4b98-b5c0-103c1ebcb3cc-53a79dff\/relevance\/1. Accessed 19 Sep 2022"},{"issue":"1","key":"11_CR6","doi-asserted-by":"publisher","first-page":"146","DOI":"10.1016\/j.joi.2010.10.002","volume":"5","author":"M Cobo","year":"2011","unstructured":"Cobo, M., L\u00f3pez-Herrera, A., Herrera-Viedma, E., Herrera, F.: An approach for detecting, quantifying, and visualizing the evolution of a research field: a practical application to the fuzzy sets theory field. J. Informetrics 5(1), 146\u2013166 (2011). https:\/\/doi.org\/10.1016\/j.joi.2010.10.002","journal-title":"J. Informetrics"},{"key":"11_CR7","doi-asserted-by":"crossref","unstructured":"da Cruz Alves, N., Wangenheim, C.G.V., Hauck, J.C.R.: Approaches to assess computational thinking competences based on code analysis in k-12 education: a systematic mapping study. Inf. Educ. 18(1), 17\u201339 (2019). https:\/\/doi.org\/10.15388\/infedu.2019.02","DOI":"10.15388\/infedu.2019.02"},{"key":"11_CR8","doi-asserted-by":"publisher","unstructured":"DeNero, J., Sridhara, S., P\u00e9rez-Qui\u00f1ones, M., Nayak, A., Leong, B.: Beyond autograding: advances in student feedback platforms. In: Proceedings of the 2017 ACM SIGCSE Technical Symposium on Computer Science Education. SIGCSE 2017, Association for Computing Machinery, New York, pp. 651\u2013652 (2017). https:\/\/doi.org\/10.1145\/3017680.3017686","DOI":"10.1145\/3017680.3017686"},{"key":"11_CR9","doi-asserted-by":"publisher","unstructured":"Edwards, S.H., Shams, Z.: Comparing test quality measures for assessing student-written tests. In: Companion Proceedings of the 36th International Conference on Software Engineering. ICSE Companion 2014, Association for Computing Machinery, New York, pp. 354\u2013363. (2014). https:\/\/doi.org\/10.1145\/2591062.2591164","DOI":"10.1145\/2591062.2591164"},{"key":"11_CR10","doi-asserted-by":"publisher","unstructured":"Falkner, N., Vivian, R., Piper, D., Falkner, K.: Increasing the effectiveness of automated assessment by increasing marking granularity and feedback units. In: Proceedings of the 45th ACM Technical Symposium on Computer Science Education. SIGCSE 2014, Association for Computing Machinery, New York, pp. 9\u201314 (2014). https:\/\/doi.org\/10.1145\/2538862.2538896","DOI":"10.1145\/2538862.2538896"},{"issue":"2","key":"11_CR11","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1177\/0165551504042802","volume":"30","author":"E Garfield","year":"2004","unstructured":"Garfield, E.: Historiographic mapping of knowledge domains literature. J. Inf. Sci. 30(2), 119\u2013145 (2004). https:\/\/doi.org\/10.1177\/0165551504042802","journal-title":"J. Inf. Sci."},{"key":"11_CR12","doi-asserted-by":"publisher","unstructured":"Ihantola, P., Ahoniemi, T., Karavirta, V., Sepp\u00e4l\u00e4, O.: Review of recent systems for automatic assessment of programming assignments. In: Proceedings of the 10th Koli Calling International Conference on Computing Education Research - Koli Calling 2010, pp. 86\u201393. ACM Press, Berlin, Germany (2010). https:\/\/doi.org\/10.1145\/1930464.1930480","DOI":"10.1145\/1930464.1930480"},{"key":"11_CR13","doi-asserted-by":"publisher","unstructured":"Insa, D., Silva, J.: Semi-automatic assessment of unrestrained java code: a library, a DSL, and a workbench to assess exams and exercises. In: Proceedings of the 2015 ACM Conference on Innovation and Technology in Computer Science Education. ITiCSE 2015, Association for Computing Machinery, New York, pp. 39\u201344 (2015). https:\/\/doi.org\/10.1145\/2729094.2742615","DOI":"10.1145\/2729094.2742615"},{"key":"11_CR14","doi-asserted-by":"publisher","unstructured":"Monperrus, M.: A critical review of automatic patch generation learned from human-written patches: essay on the problem statement and the evaluation of automatic software repair. In: Proceedings of the 36th International Conference on Software Engineering. ICSE 2014, Association for Computing Machinery, New York, pp. 234\u2013242 (2014). https:\/\/doi.org\/10.1145\/2568225.2568324","DOI":"10.1145\/2568225.2568324"},{"key":"11_CR15","doi-asserted-by":"publisher","unstructured":"Moon, S., Kim, Y., Kim, M., Yoo, S.: Ask the mutants: mutating faulty programs for fault localization. In: 2014 IEEE Seventh International Conference on Software Testing, Verification and Validation, pp. 153\u2013162. IEEE (2014). https:\/\/doi.org\/10.1109\/ICST.2014.28","DOI":"10.1109\/ICST.2014.28"},{"issue":"5","key":"11_CR16","doi-asserted-by":"publisher","first-page":"2901","DOI":"10.1007\/s10664-017-9550-0","volume":"23","author":"M Motwani","year":"2017","unstructured":"Motwani, M., Sankaranarayanan, S., Just, R., Brun, Y.: Do automated program repair techniques repair hard and important bugs? Empirical Softw. Eng. 23(5), 2901\u20132947 (2017). https:\/\/doi.org\/10.1007\/s10664-017-9550-0","journal-title":"Empirical Softw. Eng."},{"issue":"2","key":"11_CR17","doi-asserted-by":"publisher","first-page":"545","DOI":"10.1016\/j.compedu.2009.09.005","volume":"54","author":"KA Naud\u00e9","year":"2010","unstructured":"Naud\u00e9, K.A., Greyling, J.H., Vogts, D.: Marking student programs using graph similarity. Comput. Educ. 54(2), 545\u2013561 (2010). https:\/\/doi.org\/10.1016\/j.compedu.2009.09.005","journal-title":"Comput. Educ."},{"key":"11_CR18","doi-asserted-by":"publisher","unstructured":"Paiva, J.C., Leal, J.P., Figueira, A.: Automated assessment in computer science education: a state-of-the-art review. ACM Trans. Comput. Educ. 22(3) (2022). https:\/\/doi.org\/10.1145\/3513140","DOI":"10.1145\/3513140"},{"key":"11_CR19","doi-asserted-by":"publisher","unstructured":"Pape, S., Flake, J., Beckmann, A., J\u00fcrjens, J.: Stage: a software tool for automatic grading of testing exercises: case study paper. In: Proceedings of the 38th International Conference on Software Engineering Companion. ICSE 2016, Association for Computing Machinery, New York, pp. 491\u2013500 (2016). https:\/\/doi.org\/10.1145\/2889160.2889203","DOI":"10.1145\/2889160.2889203"},{"key":"11_CR20","doi-asserted-by":"publisher","unstructured":"Porfirio, A., Pereira, R., Maschio, E.: Automatic source code evaluation: a systematic mapping. Technical report, Federal University of Technology, Paran\u00e1, Brazil (UTFPR) (2021). https:\/\/doi.org\/10.13140\/RG.2.2.36112.33287","DOI":"10.13140\/RG.2.2.36112.33287"},{"key":"11_CR21","doi-asserted-by":"publisher","unstructured":"Singh, R., Gulwani, S., Solar-Lezama, A.: Automated feedback generation for introductory programming assignments. In: Proceedings of the 34th ACM SIGPLAN Conference on Programming Language Design and Implementation. PLDI 2013, Association for Computing Machinery, New York, pp. 15\u201326 (2013). https:\/\/doi.org\/10.1145\/2491956.2462195","DOI":"10.1145\/2491956.2462195"},{"key":"11_CR22","doi-asserted-by":"publisher","unstructured":"Smith, R., Tang, T., Warren, J., Rixner, S.: An automated system for interactively learning software testing. In: Proceedings of the 2017 ACM Conference on Innovation and Technology in Computer Science Education. ITiCSE 2017, Association for Computing Machinery, New York, pp. 98\u2013103 (2017). https:\/\/doi.org\/10.1145\/3059009.3059022","DOI":"10.1145\/3059009.3059022"},{"key":"11_CR23","doi-asserted-by":"publisher","unstructured":"Souza, D.M., Felizardo, K.R., Barbosa, E.F.: A systematic literature review of assessment tools for programming assignments. In: 2016 IEEE 29th International Conference on Software Engineering Education and Training (CSEET), pp. 147\u2013156. IEEE (2016). https:\/\/doi.org\/10.1109\/CSEET.2016.48","DOI":"10.1109\/CSEET.2016.48"},{"issue":"1","key":"11_CR24","doi-asserted-by":"publisher","first-page":"60","DOI":"10.1504\/IJKL.2015.071054","volume":"10","author":"DMD Souza","year":"2015","unstructured":"Souza, D.M.D., Isotani, S., Barbosa, E.F.: Teaching novice programmers using progtest. Int. J. Knowl. Learn. 10(1), 60\u201377 (2015). https:\/\/doi.org\/10.1504\/IJKL.2015.071054","journal-title":"Int. J. Knowl. Learn."},{"key":"11_CR25","doi-asserted-by":"publisher","unstructured":"Sridhara, S., Hou, B., Lu, J., DeNero, J.: Fuzz testing projects in massive courses. In: Proceedings of the Third (2016) ACM Conference on Learning @ Scale. L@S 2016, Association for Computing Machinery, New York, pp. 361\u2013367 (2016). https:\/\/doi.org\/10.1145\/2876034.2876050","DOI":"10.1145\/2876034.2876050"},{"key":"11_CR26","doi-asserted-by":"publisher","unstructured":"Srikant, S., Aggarwal, V.: A system to grade computer programming skills using machine learning. In: Proceedings of the 20th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. KDD 2014, Association for Computing Machinery, New York, pp. 1887\u20131896 (2014). https:\/\/doi.org\/10.1145\/2623330.2623377","DOI":"10.1145\/2623330.2623377"},{"issue":"1","key":"11_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.compedu.2011.08.015","volume":"58","author":"E Verd\u00fa","year":"2012","unstructured":"Verd\u00fa, E., Regueras, L.M., Verd\u00fa, M.J., Leal, J.P., de Castro, J.P., Queir\u00f3s, R.: A distributed system for learning programming on-line. Comput. Educ. 58(1), 1\u201310 (2012). https:\/\/doi.org\/10.1016\/j.compedu.2011.08.015","journal-title":"Comput. Educ."},{"key":"11_CR28","doi-asserted-by":"publisher","unstructured":"von Wangenheim, C.G., et al.: Codemaster - automatic assessment and grading of app inventor and snap! programs. Inf. Educ. 17(1), 117\u2013150 (2018). https:\/\/doi.org\/10.15388\/infedu.2018.08","DOI":"10.15388\/infedu.2018.08"},{"key":"11_CR29","doi-asserted-by":"publisher","unstructured":"Wen, M., Chen, J., Wu, R., Hao, D., Cheung, S.C.: Context-aware patch generation for better automated program repair. In: Proceedings of the 40th International Conference on Software Engineering. ICSE 2018, Association for Computing Machinery, New York, pp. 1\u201311 (2018). https:\/\/doi.org\/10.1145\/3180155.3180233","DOI":"10.1145\/3180155.3180233"},{"key":"11_CR30","doi-asserted-by":"publisher","unstructured":"Xiong, Y., et al.: Precise condition synthesis for program repair. In: Proceedings of the 39th International Conference on Software Engineering. ICSE 2017, pp. 416\u2013426. IEEE Press (2017). https:\/\/doi.org\/10.1109\/ICSE.2017.45","DOI":"10.1109\/ICSE.2017.45"},{"issue":"2","key":"11_CR31","doi-asserted-by":"publisher","first-page":"277","DOI":"10.1007\/s10489-016-0892-x","volume":"47","author":"R Yera","year":"2016","unstructured":"Yera, R., Mart\u00ednez, L.: A recommendation approach for programming online judges supported by data preprocessing techniques. Appl. Intell. 47(2), 277\u2013290 (2016). https:\/\/doi.org\/10.1007\/s10489-016-0892-x","journal-title":"Appl. Intell."}],"container-title":["Lecture Notes in Computer Science","Learning Technologies and Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-33023-0_11","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,1]],"date-time":"2023-09-01T16:04:06Z","timestamp":1693584246000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-33023-0_11"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031330223","9783031330230"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-33023-0_11","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"26 May 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICWL","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Web-Based Learning","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Tenerife","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Spain","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21 November 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23 November 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icwl2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/www.icwl-sete.eu\/index.html","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"double blind","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"82","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"45","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"55% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"presented together with SETE 2022 - joint volume with ICWL and SETE 2022","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}