{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,18]],"date-time":"2026-04-18T12:42:22Z","timestamp":1776516142533,"version":"3.51.2"},"reference-count":192,"publisher":"Association for Computing Machinery (ACM)","issue":"10","funder":[{"DOI":"10.13039\/501100002428","name":"Austrian Science Fund","doi-asserted-by":"publisher","award":["V 745-N"],"award-info":[{"award-number":["V 745-N"]}],"id":[{"id":"10.13039\/501100002428","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["ACM Comput. Surv."],"published-print":{"date-parts":[[2026,7,31]]},"abstract":"<jats:p>\n                    <jats:italic toggle=\"yes\">Semantic resources<\/jats:italic>\n                    such as ontologies and knowledge graphs enable intelligent applications in a broad range of application areas. As ensuring their quality is important,\n                    <jats:italic toggle=\"yes\">evaluation<\/jats:italic>\n                    activities are an integral part of their life-cycle. Yet, several aspects of these resources, such as domain or modelling correctness, are not amenable to automatic evaluation and require assessment through human participation. Despite such\n                    <jats:italic toggle=\"yes\">human-centric evaluation of semantic resources (HESR)<\/jats:italic>\n                    is essential, there is a lack of a systematic understanding of this area in terms of a\n                    <jats:italic toggle=\"yes\">theoretical framework<\/jats:italic>\n                    as well as an\n                    <jats:italic toggle=\"yes\">overview of concrete HESR approaches and trends<\/jats:italic>\n                    . We address this gap by, firstly, integrating existing literature into a theoretical framework for defining and characterising HESR thus developing a new perspective on this area. Second, we perform a Systematic Mapping Study of 15 years of work published in this area covering 144 papers in order to empirically ground and extend the theoretical framework and identify trends and best practices. These\n                    <jats:italic toggle=\"yes\">theoretical foundations<\/jats:italic>\n                    ,\n                    <jats:italic toggle=\"yes\">insights into current trends and practices<\/jats:italic>\n                    as well as our\n                    <jats:italic toggle=\"yes\">openly available data and analysis scripts<\/jats:italic>\n                    aim to enable new research in this area and offer practical guidelines for performing HESR to researchers and practitioners alike.\n                  <\/jats:p>\n                  <jats:p\/>\n                  <jats:p\/>","DOI":"10.1145\/3800939","type":"journal-article","created":{"date-parts":[[2026,3,5]],"date-time":"2026-03-05T21:42:19Z","timestamp":1772746939000},"page":"1-38","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Human-centric Evaluation of Semantic Resources: A Systematic Mapping Study"],"prefix":"10.1145","volume":"58","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9301-8418","authenticated-orcid":false,"given":"Marta","family":"Sabou","sequence":"first","affiliation":[{"name":"Vienna University of Economics and Business","place":["Wien, Austria"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0895-6379","authenticated-orcid":false,"given":"Stefani","family":"Tsaneva","sequence":"additional","affiliation":[{"name":"Vienna University of Economics and Business","place":["Wien, Austria"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5939-4321","authenticated-orcid":false,"given":"Miriam","family":"Fernandez","sequence":"additional","affiliation":[{"name":"The Open University","place":["Milton Keynes, United Kingdom of Great Britain and Northern Ireland"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3587-0367","authenticated-orcid":false,"given":"Mar\u00eda","family":"Poveda-Villal\u00f3n","sequence":"additional","affiliation":[{"name":"Universidad Polit\u00e9cnica de Madrid","place":["Madrid, Spain"]}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3807-5019","authenticated-orcid":false,"given":"Mari Carmen","family":"Su\u00e1rez-Figueroa","sequence":"additional","affiliation":[{"name":"Universidad Polit\u00e9cnica de Madrid","place":["Madrid, Spain"]}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,4,18]]},"reference":[{"key":"e_1_3_3_2_2","doi-asserted-by":"publisher","unstructured":"M. Acosta A. Zaveri E. Simperl D. Kontokostas F. Fl\u00f6ck and J. Lehmann. 2016. Detecting linked data quality issues via crowdsourcing: A DBpedia study. Semantic Web 9 3 (2016) 303\u2013335. DOI:10.3233\/sw-160239","DOI":"10.3233\/sw-160239"},{"key":"e_1_3_3_3_2","doi-asserted-by":"crossref","unstructured":"N. Al-Rajebah and H. Al-Khalifa. 2014. Extracting ontologies from Arabic Wikipedia: A linguistic approach. Arabian journal for Science and Engineering 39 4 (2014) 2749\u20132771.","DOI":"10.1007\/s13369-013-0791-y"},{"key":"e_1_3_3_4_2","doi-asserted-by":"crossref","unstructured":"L. Alkhariji S. De O. Rana and C. Perera. 2023. Semantics-based privacy by design for Internet of Things applications. Future Generation Computer Systems 138 1 (2023) 280\u2013295.","DOI":"10.1016\/j.future.2022.08.013"},{"key":"e_1_3_3_5_2","doi-asserted-by":"publisher","unstructured":"M. Amith Z. He J. Bian J. A. Lossio-Ventura and C. Tao. 2018. Assessing the practice of biomedical ontology evaluation: Gaps and opportunities. Journal of Biomedical Informatics 80 1 (2018) 1\u201313. DOI:10.1016\/j.jbi.2018.02.010","DOI":"10.1016\/j.jbi.2018.02.010"},{"key":"e_1_3_3_6_2","doi-asserted-by":"publisher","unstructured":"M. Amith F. J. Manion M. R. Harris Y. Zhang H. Xu and C. Tao. 2017. Expressing biomedical ontologies in natural language for expert evaluation. Studies in Health Technology and Informatics 245 1 (2017) 838\u2013842. DOI:10.3233\/978-1-61499-830-3-838","DOI":"10.3233\/978-1-61499-830-3-838"},{"key":"e_1_3_3_7_2","doi-asserted-by":"publisher","unstructured":"N. Anand J. H. V. Duin and L. Tavasszy. 2016. Framework for modelling multi-stakeholder city logistics domain using the agent based modelling approach. Transportation Research Procedia 16 1 (2016) 4\u201315. DOI:10.1016\/j.trpro.2016.11.002","DOI":"10.1016\/j.trpro.2016.11.002"},{"key":"e_1_3_3_8_2","doi-asserted-by":"crossref","unstructured":"D. Ar\u00e9chiga F. Crestani and J. Vegas. 2014. Ontology-driven word recommendation for mobile Web search. The Knowledge Engineering Review 29 2 (2014) 186\u2013200.","DOI":"10.1017\/S0269888914000125"},{"key":"e_1_3_3_9_2","volume-title":"Proceedings of the International Conference on Process Automation, Control, and Computing","author":"Aruna T.","year":"2011","unstructured":"T. Aruna, K. Saranya, and C. Bhandari. 2011. A survey on ontology evaluation tools. In Proceedings of the International Conference on Process Automation, Control, and Computing."},{"key":"e_1_3_3_10_2","first-page":"196","volume-title":"Proceedings of the AI in Medicine","author":"Abacha A. B.","year":"2013","unstructured":"A. B. Abacha, M. Da Silveira, and C. Pruski. 2013. Medical ontology validation through question answering. In Proceedings of the AI in Medicine. 196\u2013205."},{"key":"e_1_3_3_11_2","doi-asserted-by":"publisher","unstructured":"A. B. Abacha and P. Zweigenbaum. 2015. MEANS: A medical question-answering system combining NLP techniques and semantic Web technologies. Information Processing and Management 51 5 (2015) 570\u2013594. DOI:10.1016\/j.ipm.2015.04.006","DOI":"10.1016\/j.ipm.2015.04.006"},{"key":"e_1_3_3_12_2","volume-title":"The Description Logic Handbook: Theory, Implementation and Applications","author":"Baader F.","year":"2003","unstructured":"F. Baader. 2003. The Description Logic Handbook: Theory, Implementation and Applications. Cambridge University Press."},{"key":"e_1_3_3_13_2","volume-title":"Proceedings of the IEEE European Symposium on Security and Privacy Workshops","author":"Babayeva G.","year":"2022","unstructured":"G. Babayeva, K. Maennel, and O. M. Maennel. 2022. Building an ontology for cyber defence exercises. In Proceedings of the IEEE European Symposium on Security and Privacy Workshops."},{"key":"e_1_3_3_14_2","first-page":"177","volume-title":"Proceedings of the IEEE 30th International Conference on Enabling Technologies: Infrastructure for Collaborative Enterprises","author":"Baghernezhad-Tabasi S.","year":"2021","unstructured":"S. Baghernezhad-Tabasi, L. Druette, F. Jouanot, C. Meurger, and M. Rousset. 2021. OntoSAMSEI: Interactive ontology engineering for supporting simulation-based training in Medicine. In Proceedings of the IEEE 30th International Conference on Enabling Technologies: Infrastructure for Collaborative Enterprises. IEEE, 177\u2013182."},{"key":"e_1_3_3_15_2","doi-asserted-by":"publisher","unstructured":"M. C Ballandies and E. Pournaras. 2021. Mobile link prediction: Automated creation and crowdsourced validation of knowledge graphs. Microprocessors and Microsystems 87 1 (2021) 104335. DOI:10.1016\/j.micpro.2021.104335","DOI":"10.1016\/j.micpro.2021.104335"},{"key":"e_1_3_3_16_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-47721-8_6"},{"key":"e_1_3_3_17_2","doi-asserted-by":"publisher","unstructured":"Y. S. Bareedu T. Fr\u00fchwirth C. Niedermeier M. Sabou G. Steindl A. S. Thuluva and N. Tsaneva S.and Tufek Ozkaya. 2024. Deriving semantic validation rules from industrial standards: An OPC UA study. Semantic Web 15 2 (2024) 517\u2013554. DOI:10.3233\/SW-233342","DOI":"10.3233\/SW-233342"},{"key":"e_1_3_3_18_2","unstructured":"R. Barros P. Kislansky L. Salvador R. Almeida M. Breyer L. Pedraza and V. Vieira. 2015. EDXL-RESCUER ontology: An update based on faceted taxonomy approach. CEUR Workshop Proceedings 1442 1 (2015)."},{"key":"e_1_3_3_19_2","doi-asserted-by":"crossref","unstructured":"M. Bellucci N. Delestre N. Malandain and C. Zanni-Merk. 2024. Towards counterfactual explanations for ontologies. Semantic Web 15 5 (2024) 1611\u20131636.","DOI":"10.3233\/SW-243566"},{"key":"e_1_3_3_20_2","doi-asserted-by":"publisher","unstructured":"E. Bernasconi M. Ceriani M. Mecella and T. Catarci. 2023. Design realization and user evaluation of the ARCA system for exploring a digital library. International Journal on Digital Libraries 24 1 (2023) 1\u201322. DOI:10.1007\/s00799-022-00343-0","DOI":"10.1007\/s00799-022-00343-0"},{"key":"e_1_3_3_21_2","doi-asserted-by":"crossref","first-page":"205","DOI":"10.4018\/978-1-60960-593-3.ch008","volume-title":"Proceedings of the Semantic Services, Interoperability and Web Applications: Emerging Concepts.","author":"Bizer C.","year":"2011","unstructured":"C. Bizer, T. Heath, and T. Berners-Lee. 2011. Linked data: The story so far. In Proceedings of the Semantic Services, Interoperability and Web Applications: Emerging Concepts.IGI global, 205\u2013227."},{"key":"e_1_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.1109\/COMPSAC54236.2022.00145"},{"key":"e_1_3_3_23_2","unstructured":"T. Bobic J. Waitelonis and H. Sack. 2015. Franco - A ground truth corpus for fact ranking evaluation. CEUR Workshop Proceedings 1556 1 (2015)."},{"key":"e_1_3_3_24_2","doi-asserted-by":"crossref","unstructured":"R. Bouzidi A. De Nicola F. Nader and R. Chalal. 2019. OntoGamif: A modular ontology for integrated gamification. Applied Ontology 14 3 (2019) 215\u2013249.","DOI":"10.3233\/AO-190212"},{"key":"e_1_3_3_25_2","volume-title":"Proceedings of the Conference on Data Mining and Data Warehouses","author":"Brank J.","year":"2005","unstructured":"J. Brank, M. Grobelnik, and D. Mladeni\u0107. 2005. A survey of ontology evaluation techniques. In Proceedings of the Conference on Data Mining and Data Warehouses."},{"key":"e_1_3_3_26_2","doi-asserted-by":"publisher","unstructured":"A. Breit L. Waltersdorfer F. J. Ekaputra M. Sabou A. Ekelhart A. Iana H. Paulheim J. Portisch A. Revenko A. t. Teije et\u00a0al. 2023. Combining machine learning and semantic web: A systematic mapping study. ACM Computing Surveys 55 14 (2023) 1\u201341. DOI:10.1145\/3586163","DOI":"10.1145\/3586163"},{"key":"e_1_3_3_27_2","doi-asserted-by":"publisher","unstructured":"P. Brereton B. A. Kitchenham D. Budgen M. Turner and M. Khalil. 2007. Lessons from applying the systematic literature review process within the software engineering domain. Journal of Systems and Software 80 4 (2007) 571\u2013583. DOI:10.1016\/j.jss.2006.07.009","DOI":"10.1016\/j.jss.2006.07.009"},{"key":"e_1_3_3_28_2","volume-title":"International Conference Language Resources and Evaluation","author":"Brewster C.","year":"2004","unstructured":"C. Brewster, H. Alani, S. Dasmahapatra, and Y. Wilks. 2004. Data driven ontology evaluation. In International Conference Language Resources and Evaluation."},{"key":"e_1_3_3_29_2","doi-asserted-by":"publisher","unstructured":"T. J. Bright E. Yoko Furuya G. J. Kuperman J. J. Cimino and S. Bakken. 2012. Development and evaluation of an ontology for guiding appropriate antibiotic prescribing. Journal of Biomedical Informatics 45 1 (2012) 120\u2013128. DOI:10.1016\/j.jbi.2011.10.001","DOI":"10.1016\/j.jbi.2011.10.001"},{"key":"e_1_3_3_30_2","doi-asserted-by":"publisher","unstructured":"R. Brownlow S. Capuzzi S. Helmer L. Martins I. Normann and A. Poulovassilis. 2015. An ontological approach to creating an Andean weaving knowledge base. Journal on Computing and Cultural Heritage 8 2 (2015) 31 pages. DOI:10.1145\/2700427","DOI":"10.1145\/2700427"},{"key":"e_1_3_3_31_2","doi-asserted-by":"crossref","unstructured":"H. Bu and K. Kuwabara. 2021. Validating knowledge contents with blockchain-assisted gamified crowdsourcing. Vietnam Journal of Computer Science 8 4 (2021) 471\u2013491.","DOI":"10.1142\/S2196888821500202"},{"key":"e_1_3_3_32_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICIPMC58929.2023.00030"},{"key":"e_1_3_3_33_2","doi-asserted-by":"publisher","unstructured":"D. Budgen P. Brereton S. Drummond and N. Williams. 2018. Reporting systematic reviews: Some lessons from a tertiary study. Information and Software Technology 95 1 (2018) 62\u201374. DOI:10.1016\/j.infsof.2017.10.017","DOI":"10.1016\/j.infsof.2017.10.017"},{"key":"e_1_3_3_34_2","doi-asserted-by":"crossref","unstructured":"A. CB K. Mahesh and N. Sanda. 2023. Ontology-based semantic data interestingness using BERT models. Connection Science 35 1 (2023) 2190499.","DOI":"10.1080\/09540091.2023.2190499"},{"key":"e_1_3_3_35_2","doi-asserted-by":"crossref","unstructured":"I. Celino G. Calegari and A. Fiano. 2020. Refining linked data with games with a purpose. Data Intelligence 2 3 (2020) 417\u2013442.","DOI":"10.1162\/dint_a_00056"},{"key":"e_1_3_3_36_2","doi-asserted-by":"crossref","unstructured":"P. Charlton G. Magoulas and D. Laurillard. 2012. Enabling creative learning design through semantic technologies. Technology Pedagogy and Education 21 2 (2012) 231\u2013253.","DOI":"10.1080\/1475939X.2012.698165"},{"key":"e_1_3_3_37_2","doi-asserted-by":"publisher","unstructured":"Y. Chen X. Peng B. Zhong and H. Luo. 2016. Application of ontology in vulnerability analysis of metro operation systems. Structure and Infrastructure Engineering 12 10 (2016) 1256\u20131266. DOI:10.1080\/15732479.2015.1110602","DOI":"10.1080\/15732479.2015.1110602"},{"key":"e_1_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICNIDC.2018.8525665"},{"key":"e_1_3_3_39_2","unstructured":"B. C. K. Choi and A. W. P. Pak. 2005. Peer reviewed: A catalog of biases in questionnaires. Preventing Chronic Disease 2 1 (2005) A13."},{"key":"e_1_3_3_40_2","doi-asserted-by":"publisher","unstructured":"C. C. Chou A. P. Jeng C. P. Chu C. H. Chang and R. G. Wang. 2018. Generation and visualization of earthquake drill scripts for first responders using ontology and serious game platforms. Advanced Engineering Informatics 38 1 (2018) 538\u2013554. DOI:10.1016\/j.aei.2018.09.003","DOI":"10.1016\/j.aei.2018.09.003"},{"key":"e_1_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2749431"},{"key":"e_1_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-07443-6_39"},{"key":"e_1_3_3_43_2","doi-asserted-by":"publisher","unstructured":"F. Corno L. De Russis and A. Monge Roffarello. 2019. A high-level semantic approach to End-User Development in the Internet of Things. International Journal of Human Computer Studies 125 1 (2019) 41\u201354. DOI:10.1016\/j.ijhcs.2018.12.008","DOI":"10.1016\/j.ijhcs.2018.12.008"},{"key":"e_1_3_3_44_2","doi-asserted-by":"publisher","unstructured":"Y. Dang F. Li X. Hu V. K. Keloth M. Zhang S. Fu M. F. Amith J. W. Fan J. Du E. Yu et\u00a0al. 2023. Systematic design and data-driven evaluation of social determinants of health ontology (SDoHO). Journal of the American Medical Informatics Association 30 9 (2023) 1465\u20131473. DOI:10.1093\/jamia\/ocad096","DOI":"10.1093\/jamia\/ocad096"},{"key":"e_1_3_3_45_2","doi-asserted-by":"crossref","first-page":"170","DOI":"10.1007\/978-3-319-38791-8_10","volume-title":"Proceedings of the Web Engineering","author":"Darari F.","year":"2016","unstructured":"F. Darari, S. Razniewski, R. E. Prasojo, and W. Nutt. 2016. Enabling fine-grained RDF data completeness assessment. In Proceedings of the Web Engineering. 170\u2013187."},{"key":"e_1_3_3_46_2","doi-asserted-by":"crossref","unstructured":"F. D. Davis. 1989. Perceived usefulness perceived ease of use and user acceptance of information technology. MIS Quarterly 13 3 (1989) 319\u2013340.","DOI":"10.2307\/249008"},{"key":"e_1_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.1145\/3132218.3132219"},{"key":"e_1_3_3_48_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-62466-8_37"},{"key":"e_1_3_3_49_2","doi-asserted-by":"publisher","unstructured":"G. Demartini D. E. Difallah and P. Cudr\u00e9-Mauroux. 2013. Large-scale linked data integration using probabilistic reasoning and crowdsourcing. The VLDB Journal 22 5 (2013) 665\u2013687. DOI:10.1007\/s00778-013-0324-z","DOI":"10.1007\/s00778-013-0324-z"},{"key":"e_1_3_3_50_2","doi-asserted-by":"publisher","unstructured":"C. Di Francescomarino M. Rospocher C. Ghidini and A. Valerio. 2014. The role of semantic annotations in business process modelling. In Proceedings of the IEEE 18th International Enterprise Distributed Object Computing Conference.181\u2013189. DOI:10.1109\/EDOC.2014.33","DOI":"10.1109\/EDOC.2014.33"},{"key":"e_1_3_3_51_2","doi-asserted-by":"crossref","unstructured":"A. Doan R. Ramakrishnan and A. Y. Halevy. 2011. Crowdsourcing systems on the world-wide web. Communications of the ACM 54 4 (2011) 86\u201396.","DOI":"10.1145\/1924421.1924442"},{"key":"e_1_3_3_52_2","doi-asserted-by":"publisher","unstructured":"J. D\u2019Souza H. Hussein J. Evans L. Vogt O. Karras V. Ilangovan A. Lorenz and S. Auer. 2024. Quality assessment of research comparisons in the open research knowledge graph: A case study. JLIS. it 15 1 (2024) 126\u2013143. DOI:10.36253\/jlis.it-547","DOI":"10.36253\/jlis.it-547"},{"key":"e_1_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/3611643.3616317"},{"key":"e_1_3_3_54_2","unstructured":"A. Dumitrache L. Aroyo and C. Welty. 2015. Achieving expert-level annotation quality with CrowdTruth: The case of medical relation extraction. CEUR Workshop Proceedings 1 (2015) 1428."},{"key":"e_1_3_3_55_2","doi-asserted-by":"publisher","unstructured":"K. Eckert M. Niepert C. Niemann C. Buckner C. Allen and H. Stuckenschmidt. 2010. Crowdsourcing the assembly of concept hierarchies. Proceedings 10th Annual Joint Conference on Digital Libraries (2010) 139\u2013148. DOI:10.1145\/1816123.1816143","DOI":"10.1145\/1816123.1816143"},{"key":"e_1_3_3_56_2","doi-asserted-by":"publisher","unstructured":"S. Egami T. Kawamura K. Kozaki and A. Ohsuga. 2022. Detecting vicious cycles in urban problem knowledge graph using inference rules. Data Intelligence 4 1 (2022) 88\u2013111. DOI:10.1162\/dint_a_00113","DOI":"10.1162\/dint_a_00113"},{"key":"e_1_3_3_57_2","doi-asserted-by":"crossref","unstructured":"N. M. El-Gohary and T. E. El-Diraby. 2010. Domain ontology for processes in infrastructure and construction. Journal of Construction Engineering and Management 136 7 (2010) 730\u2013744.","DOI":"10.1061\/(ASCE)CO.1943-7862.0000178"},{"key":"e_1_3_3_58_2","doi-asserted-by":"publisher","DOI":"10.1145\/2814864.2814881"},{"key":"e_1_3_3_59_2","doi-asserted-by":"publisher","DOI":"10.1145\/2631775.2631819"},{"key":"e_1_3_3_60_2","volume-title":"The Semantic Web: Research and Applications.","author":"Gangemi A.","year":"2006","unstructured":"A. Gangemi, C. Catenacci, M. Ciaramita, and J. Lehmann. 2006. Modelling ontology evaluation and validation. In The Semantic Web: Research and Applications."},{"key":"e_1_3_3_61_2","volume-title":"Proceedings of the International Semantic Web Conference","author":"Gangemi A.","year":"2012","unstructured":"A. Gangemi, A. Nuzzolese, V. Presutti, F. Draicchio, A. Musetti, and P. Ciancarini. 2012. Automatic typing of DBpedia entities. In Proceedings of the International Semantic Web Conference."},{"key":"e_1_3_3_62_2","doi-asserted-by":"crossref","unstructured":"S. Gardasevic and M. Lamba. 2024. \u201cIt answers questions that I didn\u2019t know i had\u201d: PhD students\u2019 evaluation of an information-sharing knowledge graph. Digital Library Perspectives 40 4 (2024) 493\u2013517. DOI:DLP-02-2024-0025","DOI":"10.1108\/DLP-02-2024-0025"},{"key":"e_1_3_3_63_2","doi-asserted-by":"crossref","unstructured":"A. P. Getman and V. V. Karasiuk. 2014. A crowdsourcing approach to building a legal ontology from text. AI and Law 22 3 (2014) 313\u2013335.","DOI":"10.1007\/s10506-014-9159-1"},{"key":"e_1_3_3_64_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-24750-0_13"},{"key":"e_1_3_3_65_2","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1007\/978-3-540-92673-3_9","volume-title":"Proceedings of the Handbook on Ontologies","author":"Guarino N.","year":"2009","unstructured":"N. Guarino and C. A. Welty. 2009. An overview of ontoclean. In Proceedings of the Handbook on Ontologies. Springer, 201\u2013220."},{"key":"e_1_3_3_66_2","doi-asserted-by":"publisher","unstructured":"H. Guo S. Gao J. Krogstie Hallvard Tr\u00e6tteberg and Alf Inge Wang. 2015. An evaluation of ontology based domain analysis for model driven development. International Journal on Semantic Web and Information Systems 11 4 (2015) 41\u201363. DOI:10.4018\/IJSWIS.2015100102","DOI":"10.4018\/IJSWIS.2015100102"},{"key":"e_1_3_3_67_2","doi-asserted-by":"crossref","unstructured":"Y. Gutierrez D. Tomas and I. Moreno. 2019. Developing an ontology schema for enriching and linking digital media assets. Future Generation Computer Systems 101 1 (2019) 381\u2013397.","DOI":"10.1016\/j.future.2019.06.023"},{"key":"e_1_3_3_68_2","doi-asserted-by":"publisher","unstructured":"S. Hao C. Chai G. Li N. Tang N. Wang and X. Yu. 2023. HOFD: An outdated fact detector for knowledge bases. IEEE Transactions on Knowledge and Data Engineering 35 10 (2023) 10775\u201310789. DOI:10.1109\/TKDE.2023.3248223","DOI":"10.1109\/TKDE.2023.3248223"},{"key":"e_1_3_3_69_2","doi-asserted-by":"crossref","unstructured":"A. R. Hevner S. T. March J. Park and S. Ram. 2004. Design science in information systems research. MIS Quarterly 28 1 (2004) 75\u2013105.","DOI":"10.2307\/25148625"},{"key":"e_1_3_3_70_2","unstructured":"B. Hitz-Gamper. 2017. What do others say about similar things - Predicate comparing for a linked data quality boost. CEUR Workshop Proceedings 1962 1 (2017)."},{"key":"e_1_3_3_71_2","doi-asserted-by":"publisher","unstructured":"A. Hogan E. Blomqvist M. Cochez C. d\u2019Amato G. D. Melo C. Gutierrez S. Kirrane J. E. L. Gayo R. Navigli S. Neumaier et\u00a0al. 2021. Knowledge graphs. ACM Computing Surveys 54 4 (2021) 1\u201337. DOI:10.1145\/3447772","DOI":"10.1145\/3447772"},{"key":"e_1_3_3_72_2","doi-asserted-by":"publisher","unstructured":"E. Horvitz. 2017. AI people and society. Science 357 6346 (2017) 7\u20137. DOI:10.1126\/science.aao2466","DOI":"10.1126\/science.aao2466"},{"key":"e_1_3_3_73_2","doi-asserted-by":"publisher","unstructured":"S. Issa O. Adekunle F. Hamdi S. Cherfi M. Dumontier and A. Zaveri. 2021. Knowledge graph completeness: A systematic literature review. IEEE Access 9 1 (2021) 31322\u201331339. DOI:10.1109\/ACCESS.2021.3056622","DOI":"10.1109\/ACCESS.2021.3056622"},{"key":"e_1_3_3_74_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-97532-6_5"},{"key":"e_1_3_3_75_2","doi-asserted-by":"publisher","unstructured":"P. Julrode and P. Jarusawat. 2025. Development of a semantic ontology for knowledge of ancient Lanna documents. ECTI Transactions on Computer and Information Technology 19 2 (2025) 168\u2013181. DOI:10.37936\/ecti-cit.2025192.256289","DOI":"10.37936\/ecti-cit.2025192.256289"},{"key":"e_1_3_3_76_2","doi-asserted-by":"publisher","unstructured":"E. A. Kardinata and N. A. Rakhmawati. 2023. Online incremental learning based on crowdsourcing for Indonesian ontology relation extraction. Inteligencia Artificial 26 72 (2023) 124\u2013136. DOI:10.4114\/intartif.vol26iss72pp124-136","DOI":"10.4114\/intartif.vol26iss72pp124-136"},{"key":"e_1_3_3_77_2","volume-title":"Proceedings of the International Conference on Privacy, Security and Trust","author":"Kaur J.","year":"2022","unstructured":"J. Kaur, R. Dara, and R. Chaturvedi. 2022. A semantic-based approach to reduce the reading time of privacy policies. In Proceedings of the International Conference on Privacy, Security and Trust."},{"key":"e_1_3_3_78_2","doi-asserted-by":"publisher","unstructured":"M. Ketterl J. Waitelonis N. Ludwig M. Knuth and H. Sack. 2011. WhoKnows? evaluating linked data heuristics with a quiz that cleans up DBpedia. Interactive Technology and Smart Education 8 4 (2011) 236\u2013248. DOI:10.1108\/17415651111189478","DOI":"10.1108\/17415651111189478"},{"key":"e_1_3_3_79_2","volume-title":"Guidelines for Performing Systematic Literature Reviews in Software Engineering","author":"Kitchenham B. A.","year":"2007","unstructured":"B. A. Kitchenham and S. Charters. 2007. Guidelines for Performing Systematic Literature Reviews in Software Engineering. Technical Report EBSE 2007-001. Keele University and Durham University Joint Report."},{"key":"e_1_3_3_80_2","doi-asserted-by":"crossref","unstructured":"B. A. Kitchenham and S. L. Pfleeger. 2002. Principles of survey research part 2: Designing a survey. ACM SIGSOFT Software Engineering Notes 27 1 (2002) 18\u201320.","DOI":"10.1145\/566493.566495"},{"key":"e_1_3_3_81_2","doi-asserted-by":"crossref","first-page":"116","DOI":"10.18653\/v1\/2023.nlrse-1.9","volume-title":"Proceedings of the 1st Workshop on Natural Language Reasoning and Structured Explanations","author":"Kobbe J.","year":"2023","unstructured":"J. Kobbe, I. Hulpu\u0219, and H. Stuckenschmidt. 2023. Effect graph: Effect relation extraction for explanation generation. In Proceedings of the 1st Workshop on Natural Language Reasoning and Structured Explanations. 116\u2013127."},{"key":"e_1_3_3_82_2","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450653"},{"key":"e_1_3_3_83_2","doi-asserted-by":"publisher","unstructured":"S. K. Kondreddi P. Triantafillou and G. Weikum. 2014. Combining information extraction and human computing for crowdsourced knowledge acquisition. In Proceedings of the IEEE 30th International Conference on Data Engineering.988\u2013999. DOI:10.1109\/ICDE.2014.6816717","DOI":"10.1109\/ICDE.2014.6816717"},{"key":"e_1_3_3_84_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-41360-5_22"},{"key":"e_1_3_3_85_2","first-page":"2470","volume-title":"Proceedings of the 22nd International Joint Conference on Artificial Intelligence","author":"Kuo Y. L.","year":"2011","unstructured":"Y. L. Kuo and J. Y. J. Hsu. 2011. Resource-bounded crowd-sourcing of commonsense knowledge. In Proceedings of the 22nd International Joint Conference on Artificial Intelligence. 2470\u20132475."},{"key":"e_1_3_3_86_2","unstructured":"Y. L. Kuo J. Y. J. Hsu and F. Shih. 2012. Contextual commonsense knowledge acquisition from social content by crowd-sourcing explanations. AAAI Workshop - Technical Report WS-12-08 1 (2012) 18\u201324."},{"key":"e_1_3_3_87_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-49381-6_4"},{"key":"e_1_3_3_88_2","doi-asserted-by":"crossref","unstructured":"P. Lambrix. 2023. Completing and debugging ontologies: State-of-the-art and challenges in repairing ontologies. ACM Journal of Data and Information Quality 15 4 (2023) 1\u201338.","DOI":"10.1145\/3597304"},{"key":"e_1_3_3_89_2","doi-asserted-by":"publisher","unstructured":"M. Lassaad H. Raja and H. H. B. Ghezala. 2020. \u201cOnto-computer-project\u201d a computer project domain ontology: Construction and validation. International Journal of Advanced Computer Science and Applications 11 3 (2020) 360\u2013366. DOI:10.14569\/ijacsa.2020.0110345","DOI":"10.14569\/ijacsa.2020.0110345"},{"key":"e_1_3_3_90_2","doi-asserted-by":"crossref","unstructured":"C. Li P. Zhao V. Sheng X. Xian J. Wu and Z. Cui. 2017. Refining automatically extracted knowledge bases using crowdsourcing. Computational Intelligence and Neuroscience 2017 1 (2017) 4092135.","DOI":"10.1155\/2017\/4092135"},{"key":"e_1_3_3_91_2","doi-asserted-by":"publisher","unstructured":"H. Li Z. Dragisic D. Faria V. Ivanova E. Jim\u00e9nez-Ruiz P. Lambrix and C. Pesquita. 2019. User validation in ontology alignment: Functional assessment and impact. The Knowledge Engineering Review 34 1 (2019) e15. DOI:10.1017\/S0269888919000080","DOI":"10.1017\/S0269888919000080"},{"key":"e_1_3_3_92_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-32236-6_72"},{"key":"e_1_3_3_93_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-35527-1_6"},{"key":"e_1_3_3_94_2","doi-asserted-by":"publisher","unstructured":"R. Lin M. Amith C. Liang R. Duan Y. Chen and C. Tao. 2018. Visualized emotion ontology: A model for representing visual cues of emotions. BMC Medical Informatics and Decision Making 18 Suppl 2 (2018) 101\u2013113. DOI:10.1186\/s12911-018-0634-6","DOI":"10.1186\/s12911-018-0634-6"},{"key":"e_1_3_3_95_2","doi-asserted-by":"crossref","unstructured":"X. Liu C. Guo and L. Zhang. 2014. Scholar metadata and knowledge generation with human and artificial intelligence. Journal of the Association for Information Science and Technology 65 6 (2014) 1187\u20131201.","DOI":"10.1002\/asi.23013"},{"key":"e_1_3_3_96_2","doi-asserted-by":"crossref","unstructured":"J. A. Lossio-Ventura W. Hogan F. Modave Y. Guo Z. He X. Yang H. Zhang and J. Bian. 2018. OC-2-KB: Integrating crowdsourcing into an obesity and cancer knowledge base curation system. BMC Medical Informatics and Decision Making 18 Suppl 2 (2018) 115\u2013127.","DOI":"10.1186\/s12911-018-0635-5"},{"key":"e_1_3_3_97_2","doi-asserted-by":"publisher","unstructured":"Y. Lou S. W. Tu C. Nyulas T. Tudorache R. J. Chalmers and M. A. Musen. 2017. Use of ontology structure and Bayesian models to aid the crowdsourcing of ICD-11 sanctioning rules. Journal of Biomedical Informatics 68 1 (2017) 20\u201334. DOI:10.1016\/j.jbi.2017.02.004","DOI":"10.1016\/j.jbi.2017.02.004"},{"key":"e_1_3_3_98_2","doi-asserted-by":"publisher","unstructured":"K. Lyu Y. Tian Y. Shang T. Zhou Z. Yang Q. Liu X. Yao P. Zhang J. Chen and J. Li. 2023. Causal knowledge graph construction and evaluation for clinical decision support of diabetic nephropathy. Journal of Biomedical Informatics 139 1 (2023) 104298. DOI:10.1016\/j.jbi.2023.104298","DOI":"10.1016\/j.jbi.2023.104298"},{"key":"e_1_3_3_99_2","unstructured":"V. Maccatrozzo L. Aroyo and W. V. Hage. 2013. Crowdsourced evaluation of semantic patterns for recommendations. CEUR Workshop Proceedings 997 1 (2013)."},{"key":"e_1_3_3_100_2","doi-asserted-by":"crossref","unstructured":"A. Maitra M. R. Kamdar D. M. Zulman M. C. Haverfield C. Brown-Johnson R. Schwartz S. T. Israni A. Verghese and M. A. Musen. 2021. Using ethnographic methods to classify the human experience in medicine: A case study of the presence ontology. Journal of the American Medical Informatics Association 28 9 (2021) 1900\u20131909.","DOI":"10.1093\/jamia\/ocab091"},{"key":"e_1_3_3_101_2","doi-asserted-by":"crossref","unstructured":"A. Malhotra M. G\u00fcndel A. M. Rajput H. T. Mevissen A. Saiz X. Pastor R. Lozano-Rubi E. H. Martinez-Lapsicina I. Zubizarreta et\u00a0al. 2015. Knowledge retrieval from pubmed abstracts and electronic medical records with the multiple sclerosis ontology. PLoS ONE 10 2 (2015) e0116718.","DOI":"10.1371\/journal.pone.0116718"},{"key":"e_1_3_3_102_2","doi-asserted-by":"publisher","unstructured":"A. Malhotra E. Younesi M. G\u00fcndel B. M\u00fcller M. T. Heneka and M. Hofmann-Apitius. 2014. ADO: A disease ontology representing the domain knowledge specific to Alzheimer\u2019s disease. Alzheimer\u2019s and Dementia 10 2 (2014) 238\u2013246. DOI:10.1016\/j.jalz.2013.02.009","DOI":"10.1016\/j.jalz.2013.02.009"},{"key":"e_1_3_3_103_2","unstructured":"T. Markotschi and J. V\u00f6lker. 2010. GuessWhat?! Human intelligence for mining linked data. CEUR Workshop Proceedings 631 1 (2010) 28\u201339."},{"key":"e_1_3_3_104_2","doi-asserted-by":"publisher","unstructured":"D. Mart\u00edn-Moncunill M. A. Sicilia-Urban E. Garc\u00eda-Barriocanal and S. S\u00e1nchez-Alonso. 2015. Evaluating the degree of domain specificity of terms in large terminologies: The case of AGROVOC. Online Information Review 39 3 (2015) 326\u2013345. DOI:10.1108\/OIR-02-2015-0052","DOI":"10.1108\/OIR-02-2015-0052"},{"key":"e_1_3_3_105_2","doi-asserted-by":"crossref","unstructured":"C. Mazo L. Salazar O. Corcho M. Trujillo and E. Alegre. 2017. A histological ontology of the human cardiovascular system. Journal of Biomedical Semantics 8 1 (2017) 47.","DOI":"10.1186\/s13326-017-0158-5"},{"key":"e_1_3_3_106_2","doi-asserted-by":"crossref","unstructured":"M. McDaniel and V. Storey. 2019. Evaluating domain ontologies: Clarification classification and challenges. ACM Computing Surveys 52 4 (2019) 1\u201344.","DOI":"10.1145\/3329124"},{"key":"e_1_3_3_107_2","volume-title":"Proceedings of the Spinning the Semantic Web","author":"McGuinness D. L.","year":"2003","unstructured":"D. L. McGuinness. 2003. Ontologies Come of Age. In Proceedings of the Spinning the Semantic Web."},{"key":"e_1_3_3_108_2","doi-asserted-by":"publisher","unstructured":"T. Mioch L. Kroon and M. A. Neerincx. 2017. Driver readiness model for regulating the transfer from automation to human control. In Proceedings of the 22nd International Conference on Intelligent User Interfaces.205\u2013213. DOI:10.1145\/3025171.3025199","DOI":"10.1145\/3025171.3025199"},{"key":"e_1_3_3_109_2","doi-asserted-by":"publisher","unstructured":"R. Monaco X. Liu T. Murino X. Cheng and P. S. Nielsen. 2023. A non-functional requirements-based ontology for supporting the development of industrial energy management systems. Journal of Cleaner Production 414 1 (2023) 137614. DOI:10.1016\/j.jclepro.2023.137614","DOI":"10.1016\/j.jclepro.2023.137614"},{"key":"e_1_3_3_110_2","volume-title":"Proceedings of the International Conference on Knowledge Engineering and Knowledge Management","author":"T. Alba C. Morales","year":"2022","unstructured":"Alba C. Morales T., E. Daga, and E. Motta. 2022. Towards a knowledge graph of health evolution. In Proceedings of the International Conference on Knowledge Engineering and Knowledge Management."},{"key":"e_1_3_3_111_2","volume-title":"CEUR Wrkshp Proceedings.","author":"Mortensen J.M.","year":"2013","unstructured":"J.M. Mortensen, M. Musen, and N. Noy. 2013. Developing crowdsourced ontology engineering tasks: An iterative process. In CEUR Wrkshp Proceedings."},{"key":"e_1_3_3_112_2","doi-asserted-by":"crossref","unstructured":"J. M. Mortensen E. P. Minty M. Januszyk T. E. Sweeney A. L. Rector N. F. Noy and M. A. Musen. 2015. Using the wisdom of the crowds to find critical errors in biomedical ontologies: A study of SNOMED CT. Journal of the American Medical Informatics Association 22 3 (2015) 640\u2013648.","DOI":"10.1136\/amiajnl-2014-002901"},{"key":"e_1_3_3_113_2","doi-asserted-by":"crossref","unstructured":"J. M. Mortensen N. Telis J. J. Hughey H. Fan-Minogue K. Van Auken M. Dumontier and M. A. Musen. 2016. Is the crowd better as an assistant or a replacement in ontology engineering? An exploration through the lens of the Gene Ontology. Journal of Biomedical Informatics 60 1 (2016) 199\u2013209.","DOI":"10.1016\/j.jbi.2016.02.005"},{"key":"e_1_3_3_114_2","doi-asserted-by":"publisher","unstructured":"S. Nazir Y. H. Motla T. Abbas A. Khatoon J. Jabeen M. Iqra and K. Bakhat. 2014. A process improvement in requirement verification and validation using ontology. Asia-Pacific World Congress on Computer Science and Engineering (2014) 1\u20138. DOI:10.1109\/APWCCSE.2014.7053837","DOI":"10.1109\/APWCCSE.2014.7053837"},{"key":"e_1_3_3_115_2","doi-asserted-by":"publisher","unstructured":"T. Nguyen S. Razniewski J. Romero and G. Weikum. 2022. Refined commonsense knowledge from large-scale web contents. IEEE Transactions on Knowledge and Data Engineering 35 8 (2022) 8431\u20138447. DOI:10.1109\/TKDE.2022.3206505","DOI":"10.1109\/TKDE.2022.3206505"},{"key":"e_1_3_3_116_2","doi-asserted-by":"crossref","unstructured":"T. Nguyen H. Takeda K. Nguyen R. Ichise and T. Cao. 2018. A novel method to predict type for DBpedia entity. Studies in Computational Intelligence 769 1 (2018) 125\u2013134.","DOI":"10.1007\/978-3-319-76081-0_11"},{"key":"e_1_3_3_117_2","doi-asserted-by":"publisher","unstructured":"S. Nikolova M. Tremaine and P. Cook. 2010. Click on bake to get cookies: Guiding word-finding with semantic associations. In Proceedings of the 12th International ACM SIGACCESS Conference on Computers and Accessibility.155\u2013162. DOI:10.1145\/1878803.1878832","DOI":"10.1145\/1878803.1878832"},{"key":"e_1_3_3_118_2","doi-asserted-by":"crossref","unstructured":"W. T. Niu and J. Kay. 2010. PERSONAF: Framework for personalised ontological reasoning in pervasive computing. UMUAI 20 1 (2010) 1\u201340.","DOI":"10.1007\/s11257-009-9068-2"},{"key":"e_1_3_3_119_2","doi-asserted-by":"publisher","DOI":"10.1145\/2464464.2464482"},{"key":"e_1_3_3_120_2","first-page":"520","volume-title":"Proceedings of the International Semantic Web Conference","author":"Nuzzolese A.G.","year":"2011","unstructured":"A.G. Nuzzolese, A. Gangemi, V. Presutti, and P. Ciancarini. 2011. Encyclopedic knowledge patterns from Wikipedia links. In Proceedings of the International Semantic Web Conference. 520\u2013536."},{"key":"e_1_3_3_121_2","doi-asserted-by":"crossref","unstructured":"A. Nuzzolese V. Presutti A. Gangemi S. Peroni and P. Ciancarini. 2017. Aemoo: Linked data exploration based on knowledge patterns. Semantic Web 8 2 (2017) 87\u2013112.","DOI":"10.3233\/SW-160222"},{"key":"e_1_3_3_122_2","volume-title":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing","author":"Ojha P.","year":"2017","unstructured":"P. Ojha and P. Talukdar. 2017. Kgeval: Accuracy estimation of automatically constructed knowledge graphs. In Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_3_123_2","volume-title":"Proceedings of the Conference on Semantic Systems.","author":"Orlandi F.","year":"2012","unstructured":"F. Orlandi, J. Breslin, and A. Passant. 2012. Aggregated, interoperable and multi-domain user profiles for the social web. In Proceedings of the Conference on Semantic Systems."},{"key":"e_1_3_3_124_2","doi-asserted-by":"publisher","unstructured":"P. T. Palomino A. M. Toda L. A. L. Rodrigues W. O. Santos L. Nacke and S. Isotani. 2023. An ontology for modelling user\u2019profiles and activities in gamified education. Research and Practice in Technology Enhanced Learning 18 1 (2023) 1\u201335. DOI:10.58459\/rptel.2023.18018","DOI":"10.58459\/rptel.2023.18018"},{"key":"e_1_3_3_125_2","doi-asserted-by":"crossref","unstructured":"C. J. Pannucci and E. G. Wilkins. 2010. Identifying and avoiding bias in research. Plastic and Reconstructive Surgery 126 2 (2010) 619.","DOI":"10.1097\/PRS.0b013e3181de24bc"},{"key":"e_1_3_3_126_2","doi-asserted-by":"crossref","unstructured":"H. Paulheim. 2017. Knowledge graph refinement: A survey of approaches and evaluation methods. Semantic Web 8 3 (2017) 489\u2013508.","DOI":"10.3233\/SW-160218"},{"key":"e_1_3_3_127_2","first-page":"180","volume-title":"Proceedings of the Semantic Web","author":"Paulheim H.","year":"2015","unstructured":"H. Paulheim and A. Gangemi. 2015. Serving DBpedia with DOLCE\u2013more than just adding a cherry on top. In Proceedings of the Semantic Web. 180\u2013196."},{"key":"e_1_3_3_128_2","doi-asserted-by":"publisher","unstructured":"E. Pavlick M. Post A. Irvine D. Kachaev and C. Callison-Burch. 2014. The language demographics of amazon mechanical turk. Transactions of the Association for Computational Linguistics 2 1 (2014) 79\u201392. DOI:10.1162\/tacl_a_00167","DOI":"10.1162\/tacl_a_00167"},{"key":"e_1_3_3_129_2","doi-asserted-by":"publisher","unstructured":"E. Pe-Than D. Goh and C. Lee. 2015. A typology of human computation games: An analysis and a review of current games. Behaviour and Information Technology 34 8 (2015) 809\u2013824. DOI:10.1080\/0144929X.2013.862304","DOI":"10.1080\/0144929X.2013.862304"},{"key":"e_1_3_3_130_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-29863-9_29"},{"key":"e_1_3_3_131_2","doi-asserted-by":"crossref","unstructured":"M. Peleg N. Veggiotti L. Sacchi and S. Wilk. 2024. How can we reward you? A compliance and reward ontology (CaRO) for eliciting quantitative reward rules for engagement in mHealth app and healthy behaviors. Journal of Biomedical Informatics 154 1 (2024) 104655.","DOI":"10.1016\/j.jbi.2024.104655"},{"key":"e_1_3_3_132_2","doi-asserted-by":"crossref","first-page":"567","DOI":"10.1007\/978-3-030-03667-6_36","volume-title":"Proceedings of the Knowledge Engineering and Knowledge Management","author":"Pesquita C.","year":"2018","unstructured":"C. Pesquita, V. Ivanova, S. Lohmann, and P. Lambrix. 2018. A framework to conduct and report on empirical user studies in semantic web contexts. In Proceedings of the Knowledge Engineering and Knowledge Management. Springer Int. Publishing, Cham, 567\u2013583."},{"key":"e_1_3_3_133_2","doi-asserted-by":"publisher","unstructured":"K. Petersen S. Vakkalanka and L. Kuzniarz. 2015. Guidelines for conducting systematic mapping studies in software engineering: An update. Information and Software Technology 64 1 (2015) 1\u201318. DOI:10.1016\/j.infsof.2015.03.007","DOI":"10.1016\/j.infsof.2015.03.007"},{"key":"e_1_3_3_134_2","unstructured":"V. Petukhova. 2014. Understanding questions and finding answers: Semantic relation annotation to compute the Expected Answer Type. International Conference Language Resources and Evaluation (2014) 44\u201352."},{"key":"e_1_3_3_135_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68288-4_32"},{"key":"e_1_3_3_136_2","doi-asserted-by":"crossref","first-page":"263","DOI":"10.5220\/0005608202630268","volume-title":"Proceedings of the 7th International Joint Conference on Knowledge Discovery, Knowledge Engineering, and Knowledge Management","author":"Pittet P.","year":"2015","unstructured":"P. Pittet and J. Barth\u00e9l\u00e9my. 2015. Exploiting users\u2019 feedbacks: Towards a task-based evaluation of application ontologies throughout their lifecycle. In Proceedings of the 7th International Joint Conference on Knowledge Discovery, Knowledge Engineering, and Knowledge Management. 263\u2013268."},{"key":"e_1_3_3_137_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICOSC.2019.8665620"},{"key":"e_1_3_3_138_2","doi-asserted-by":"publisher","unstructured":"M. Poveda-Villal\u00f3n A. G\u00f3mez-P\u00e9rez and M.C. Su\u00e1rez-Figueroa. 2014. OOPS! (ontology pitfall scanner!): An on-line tool for ontology evaluation. International Journal on Semantic Web and Information System 10 2 (2014) 7\u201334. DOI:10.4018\/ijswis.2014040102","DOI":"10.4018\/ijswis.2014040102"},{"key":"e_1_3_3_139_2","doi-asserted-by":"publisher","unstructured":"V. Presutti A. G. Nuzzolese S. Consoli A. Gangemi and D. R. Recupero. 2016. From hyperlinks to semantic web properties using open knowledge extraction. Semantic Web 7 4 (2016) 351\u2013378. DOI:10.3233\/SW-160221","DOI":"10.3233\/SW-160221"},{"key":"e_1_3_3_140_2","volume-title":"Proceedings of the IEEE International Conference on Bioinformatics and Biomedicine.","author":"Qazi K.","year":"2021","unstructured":"K. Qazi and A. Agrawal. 2021. A aontext-based crowd sourcing tool for quality assurance of SNOMED CT. In Proceedings of the IEEE International Conference on Bioinformatics and Biomedicine."},{"key":"e_1_3_3_141_2","doi-asserted-by":"publisher","DOI":"10.5220\/0005591001790186"},{"key":"e_1_3_3_142_2","doi-asserted-by":"crossref","unstructured":"Y. Raimond T. Ferne M. Smethurst and G. Adams. 2014. The BBC world service archive prototype. Journal of Web Semantics 27\u201328 1 (2014) 2\u20139.","DOI":"10.1016\/j.websem.2014.07.005"},{"key":"e_1_3_3_143_2","unstructured":"A. Rambousek A. Horak D. Klement and J. Klete\u010dka. 2017. New features in DEBVisDic for wordnet visualization and user feedback. Recent Advances in Slavonic Natural Language Processing (2017) 3\u201310."},{"key":"e_1_3_3_144_2","doi-asserted-by":"publisher","unstructured":"H. Rijgersberg M. Wigham and J. L. Top. 2011. How semantics can improve engineering processes: A case of units of measure and quantities. Advanced Engineering Informatics 25 2 (2011) 276\u2013287. DOI:10.1016\/j.aei.2010.07.008","DOI":"10.1016\/j.aei.2010.07.008"},{"key":"e_1_3_3_145_2","volume-title":"Proceedings of the Asian Conference on Intelligent Information and Database Systems.","author":"Roengsamut B.","year":"2015","unstructured":"B. Roengsamut and K. Kuwabara. 2015. Interactive refinement of linked data: Toward a crowdsourcing approach. In Proceedings of the Asian Conference on Intelligent Information and Database Systems."},{"key":"e_1_3_3_146_2","doi-asserted-by":"publisher","unstructured":"G. Rugg and P. McGeorge. 1995. Laddering. Expert Systems 12 4 (1995) 339\u2013346. DOI:10.1111\/j.1468-0394.1995.tb00271.x","DOI":"10.1111\/j.1468-0394.1995.tb00271.x"},{"key":"e_1_3_3_147_2","doi-asserted-by":"crossref","unstructured":"G. Rumin1 and I. Mekterovi\u0107. 2019. LOD construction through supervised web relation extractionand crowd validation. Journal of Web Engineering 18 1 (2019) 229\u2013255.","DOI":"10.13052\/jwe1540-9589.18137"},{"key":"e_1_3_3_148_2","first-page":"296","volume-title":"ICCCI Proceedings.","author":"Rzeniewicz J.","year":"2013","unstructured":"J. Rzeniewicz and J. Szyma\u0144ski. 2013. Bringing common sense to WordNet with a word game. In ICCCI Proceedings.296\u2013305."},{"key":"e_1_3_3_149_2","doi-asserted-by":"publisher","unstructured":"B. Sabir U. Qamar and A. W. Muzzafar. 2015. Ontology development and evaluation for urinal tract infection. IEEE Workshop on Computational Intelligence in Biometrics and Identity ManagementJanuary (2015) 195\u2013201. DOI:10.1109\/CIBIM.2014.7015463","DOI":"10.1109\/CIBIM.2014.7015463"},{"key":"e_1_3_3_150_2","doi-asserted-by":"publisher","unstructured":"M. Sabou L. Aroyo K. Bontcheva A. Bozzon and R. K. Qarout. 2018. Semantic web and human computation: The status of an emerging field. Semantic Web 9 3 (2018) 291\u2013302. DOI:10.3233\/SW-180292","DOI":"10.3233\/SW-180292"},{"key":"e_1_3_3_151_2","doi-asserted-by":"crossref","first-page":"193","DOI":"10.1007\/978-3-642-24794-1_9","volume-title":"Proceedings of the Ontology Engineering in a Networked World","author":"Sabou M.","year":"2012","unstructured":"M. Sabou and M. Fern\u00e1ndez. 2012. Ontology (network) evaluation. In Proceedings of the Ontology Engineering in a Networked World. Springer, 193\u2013212."},{"key":"e_1_3_3_152_2","doi-asserted-by":"publisher","unstructured":"M. Sabou A. Scharl and M. F\u00f6ls. 2013. Crowdsourced knowledge acquisition: Towards hybrid-genre workflows. International Journal on Semantic Web and Information Systems 9 3 (2013) 14\u201341. DOI:10.4018\/ijswis.2013070102","DOI":"10.4018\/ijswis.2013070102"},{"key":"e_1_3_3_153_2","doi-asserted-by":"publisher","unstructured":"M. Sabou S. Tsaneva M. Fern\u00e1ndez M. Poveda-Villalon and M. C. Su\u00e1rez-Figueroa. 2024. Human-centric Evaluation of Semantic Resources: A Systematic Mapping Study Protocol and Data. Zenodo. DOI:10.5281\/zenodo.10796917","DOI":"10.5281\/zenodo.10796917"},{"key":"e_1_3_3_154_2","doi-asserted-by":"publisher","DOI":"10.1145\/3184558.3191647"},{"key":"e_1_3_3_155_2","doi-asserted-by":"crossref","unstructured":"M. Sah and V. Wade. 2012. Automatic metadata mining from multilingual enterprise content. Journal of Web Semantics 11 1 (2012) 41\u201362.","DOI":"10.1016\/j.websem.2011.11.001"},{"key":"e_1_3_3_156_2","doi-asserted-by":"publisher","unstructured":"S. Sarjant C. Legg M. Stannett and D. Willcock. 2014. Crowd-sourcing ontology content and curation: The massive ontology interface. Frontiers in Artificial Intelligence and Applications 267 1 (2014) 251\u2013260. DOI:10.3233\/978-1-61499-438-1-251","DOI":"10.3233\/978-1-61499-438-1-251"},{"key":"e_1_3_3_157_2","doi-asserted-by":"publisher","unstructured":"B. Sateli F. L\u00f6ffler B. K\u00f6nig-Ries and R. Witte. 2017. ScholarLens: Extracting competences from research publications for the automatic generation of semantic user profiles. PeerJ Computer Science 3 7 (2017) e121. DOI:10.7717\/peerj-cs.121","DOI":"10.7717\/peerj-cs.121"},{"key":"e_1_3_3_158_2","doi-asserted-by":"crossref","unstructured":"M. Seadle and E. M\u00e9nard. 2012. TIIARA: The \u201cmaking of\u201d a bilingual taxonomy for retrieval of digital images. Library Hi Tech 30 4 (2012) 643\u2013654.","DOI":"10.1108\/07378831211285103"},{"key":"e_1_3_3_159_2","doi-asserted-by":"crossref","unstructured":"K. Shenoy F. Ilievski D. Garijo D. Schwabe and P. Szekely. 2022. A study of the quality of Wikidata. Journal of Web Semantics 72 1 (2022) 100679.","DOI":"10.1016\/j.websem.2021.100679"},{"key":"e_1_3_3_160_2","unstructured":"E. Simperl Q. Bu and Y. Li. 2015. Using microtasks to crowdsource DBpedia entity classification: A study in workflow design. Semantic Web 9 3 (2015) 1\u201317."},{"key":"e_1_3_3_161_2","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3556967"},{"key":"e_1_3_3_162_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2019.00039"},{"key":"e_1_3_3_163_2","doi-asserted-by":"publisher","DOI":"10.1145\/2814864.2814887"},{"key":"e_1_3_3_164_2","doi-asserted-by":"publisher","unstructured":"C. H. Tan E. Agichtein P. Ipeirotis and E. Gabrilovich. 2014. Trust but verify: Predicting contribution quality for knowledge base construction and curation. In Proceedings of the 7th ACM International Conference on Web Search and Data Mining.553\u2013562. DOI:10.1145\/2556195.2556227","DOI":"10.1145\/2556195.2556227"},{"key":"e_1_3_3_165_2","first-page":"6408","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Tandon N.","year":"2020","unstructured":"N. Tandon, K. Sakaguchi, B. Dalvi, D. Rajagopal, P. Clark, M. Guerquin, K. Richardson, and E. Hovy. 2020. A dataset for tracking entities in open domain procedural text. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing. 6408\u20136417."},{"key":"e_1_3_3_166_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-38370-0_12"},{"key":"e_1_3_3_167_2","doi-asserted-by":"crossref","unstructured":"D. Thakker S. Karanasios E. Blanchard L. Lau and V. Dimitrova. 2017. Ontology for cultural variations in interpersonal communication: Building on theoretical models and crowdsourced knowledge. Journal of the Association for Information Science and Technology 68 6 (2017) 1411\u20131428.","DOI":"10.1002\/asi.23824"},{"key":"e_1_3_3_168_2","first-page":"350","volume-title":"Proceedings of the Semantic Web","author":"Thalhammer A.","year":"2012","unstructured":"A. Thalhammer, M. Knuth, and H. Sack. 2012. Evaluating entity summarization using a game-based ground truth. In Proceedings of the Semantic Web. 350\u2013361."},{"key":"e_1_3_3_169_2","doi-asserted-by":"crossref","unstructured":"S. Tsaneva D. Dess\u00ec F. Osborne and M. Sabou. 2025. Knowledge graph validation by integrating LLMs and human-in-the-loop. IPM 62 5 (2025) 104145.","DOI":"10.1016\/j.ipm.2025.104145"},{"key":"e_1_3_3_170_2","doi-asserted-by":"crossref","unstructured":"S. Tsaneva and M. Sabou. 2024. Enhancing human-in-the-loop ontology curation results through task design. Journal of Data and Information Quality 16 1 (2024) 1\u201325.","DOI":"10.1145\/3626960"},{"key":"e_1_3_3_171_2","unstructured":"R. Vaidyambath J. Debattista N. Srivatsa and R. Brennan. 2019. An intelligent linked data quality dashboard. CEUR Workshop Proceedings 2563 1 (2019)."},{"key":"e_1_3_3_172_2","doi-asserted-by":"crossref","unstructured":"R. Van De Schoot J. De Bruin R. Schram P. Zahedi J. De Boer F. Weijdema B. Kramer M. Huijts M. Hoogerwerf G. Ferdinands et\u00a0al. 2021. An open source machine learning framework for efficient and transparent systematic reviews. Nature Machine Intelligence 3 2 (2021) 125\u2013133.","DOI":"10.1038\/s42256-020-00287-7"},{"key":"e_1_3_3_173_2","doi-asserted-by":"publisher","unstructured":"D. Vannella D. Jurgens D. Scarfini D. Toscani and R. Navigli. 2014. Validating and extending semantic knowledge bases using video games with a purpose. In Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics. 1294\u20131304. DOI:10.3115\/v1\/p14-1122","DOI":"10.3115\/v1\/p14-1122"},{"key":"e_1_3_3_174_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-77847-6_8"},{"key":"e_1_3_3_175_2","doi-asserted-by":"publisher","unstructured":"A. Vizca\u00edno F. Garc\u00eda M. Piattini and S. Beecham. 2016. A validated ontology for global software development. Computer Standards & Interfaces 46 1 (2026) 66\u201378. DOI:10.1016\/j.csi.2016.02.004","DOI":"10.1016\/j.csi.2016.02.004"},{"key":"e_1_3_3_176_2","volume-title":"Ontology Evaluation","author":"Vrande\u010di\u0107 D.","year":"2010","unstructured":"D. Vrande\u010di\u0107. 2010. Ontology Evaluation. PhD thesis. KIT, Fakult\u00e4t f\u00fcr Wirtschaftswissenschaften, Karlsruhe."},{"key":"e_1_3_3_177_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-30793-6_39"},{"key":"e_1_3_3_178_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-10983-6_44"},{"key":"e_1_3_3_179_2","first-page":"48","volume-title":"CEUR Wrkshp Proceedings.","author":"Wawrzik F.","year":"2023","unstructured":"F. Wawrzik, R. Dhouib, K. A. Rafique, and C. Grimm. 2023. Hybrid AI approach for knowledge graph construction. In CEUR Wrkshp Proceedings.48\u201363."},{"key":"e_1_3_3_180_2","doi-asserted-by":"crossref","unstructured":"A. Weichselbraun S. Gindl and A. Scharl. 2014. Enriching semantic knowledge bases for opinion mining in big data applications. Knowledge-based Systems 69 1 (2014) 78\u201385.","DOI":"10.1016\/j.knosys.2014.04.039"},{"key":"e_1_3_3_181_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.428"},{"key":"e_1_3_3_182_2","doi-asserted-by":"crossref","unstructured":"G. Wohlgenannt M. Sabou and F. Hanika. 2016. Crowd-based ontology engineering with the uComp Prot\u00e9g\u00e9 plugin. Semantic Web 7 4 (2016) 379\u2013398.","DOI":"10.3233\/SW-150181"},{"key":"e_1_3_3_183_2","doi-asserted-by":"publisher","DOI":"10.5555\/2349018"},{"key":"e_1_3_3_184_2","unstructured":"B. Xue and L. Zou. 2023. Knowledge graph quality management: A comprehensive survey. IEEE Transactions on Knowledge and Data Engineering 35 5 (2023) 4969\u20134988."},{"key":"e_1_3_3_185_2","doi-asserted-by":"crossref","unstructured":"E. Younesi A. Malhotra M. G\u00fcndel P. Scordis A. Kodamullil M. Page B. M\u00fcller S. Springstubbe U. W\u00fcllner D. Scheller et\u00a0al. 2015. PDON: Parkinson\u2019s disease ontology for representation and modeling of the Parkinson\u2019s disease knowledge domain. Theor Bio Med Model 12 1 (2015) 20.","DOI":"10.1186\/s12976-015-0017-y"},{"key":"e_1_3_3_186_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.76"},{"key":"e_1_3_3_187_2","doi-asserted-by":"publisher","unstructured":"L. Yuan Z. Huang W. Zhao and P. Stakhiyevich. 2020. Interpreting and predicting social commerce intention based on knowledge graph analysis. Electronic Commerce Research 20 1 (2020) 197\u2013222. DOI:10.1007\/s10660-019-09392-1","DOI":"10.1007\/s10660-019-09392-1"},{"key":"e_1_3_3_188_2","doi-asserted-by":"publisher","DOI":"10.1145\/2506182.2506195"},{"key":"e_1_3_3_189_2","doi-asserted-by":"crossref","unstructured":"A. Zaveri A. Rula A. Maurino R. Pietrobon J. Lehmann and S. Auer. 2015. Quality assessment for linked data: A survey. Semantic Web 7 1 (2015) 63\u201393.","DOI":"10.3233\/SW-150175"},{"key":"e_1_3_3_190_2","doi-asserted-by":"publisher","unstructured":"F. Zheng R. Abeysinghe N. Sioutos L. Whiteman L. Remennik and L. Cui. 2020. Detecting missing IS-A relations in the NCI Thesaurus using an enhanced hybrid approach. BMC Medical Informatics and Decision Making 20 Suppl 10 (2020) 273. DOI:10.1186\/s12911-020-01289-6","DOI":"10.1186\/s12911-020-01289-6"},{"key":"e_1_3_3_191_2","volume-title":"Proceedings of the IEEE International Conference on Data Engineering.","author":"Zheng L.","year":"2022","unstructured":"L. Zheng, P. Cheng, L. Chen, X. Yu, J.and Lin, and J. Yin. 2022. Crowdsourced fact validation for knowledge bases. In Proceedings of the IEEE International Conference on Data Engineering."},{"key":"e_1_3_3_192_2","doi-asserted-by":"publisher","unstructured":"M. Zhitomirsky-Geffet E. S. Erez and B. I. Judit. 2017. Toward multiviewpoint ontology construction by collaboration of non-experts and crowdsourcing: The case of the effect of diet on health. Journal of the Association for Information Science and Technology 68 3 (2017) 681\u2013694. DOI:10.1002\/asi.23686","DOI":"10.1002\/asi.23686"},{"key":"e_1_3_3_193_2","doi-asserted-by":"crossref","unstructured":"Y. Zhou S. Schockaert and J. Shah. 2019. Predicting conceptnet path quality using crowdsourced assessments of naturalness. In Proceedings of the World Wide Web Conference.","DOI":"10.1145\/3308558.3313486"}],"container-title":["ACM Computing Surveys"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3800939","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,18]],"date-time":"2026-04-18T11:47:56Z","timestamp":1776512876000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3800939"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,18]]},"references-count":192,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2026,7,31]]}},"alternative-id":["10.1145\/3800939"],"URL":"https:\/\/doi.org\/10.1145\/3800939","relation":{},"ISSN":["0360-0300","1557-7341"],"issn-type":[{"value":"0360-0300","type":"print"},{"value":"1557-7341","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,4,18]]},"assertion":[{"value":"2024-03-20","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2026-02-13","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2026-04-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}