{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,5]],"date-time":"2026-06-05T05:08:26Z","timestamp":1780636106683,"version":"3.54.1"},"publisher-location":"Cham","reference-count":31,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031434570","type":"print"},{"value":"9783031434587","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-43458-7_49","type":"book-chapter","created":{"date-parts":[[2023,10,20]],"date-time":"2023-10-20T04:02:21Z","timestamp":1697774541000},"page":"310-320","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["Evaluating Knowledge Graphs with\u00a0Hybrid Intelligence"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0895-6379","authenticated-orcid":false,"given":"Stefani","family":"Tsaneva","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,10,21]]},"reference":[{"key":"49_CR1","doi-asserted-by":"publisher","unstructured":"Acosta, M., Zaveri, A., Simperl, E., Kontokostas, D., Auer, S., Lehmann, J.: Crowdsourcing linked data quality assessment. In: Alani, H., et al. (eds.) ISWC 2013. LNCS, vol. 8219, pp. 260\u2013276. Springer, Heidelberg (2013). https:\/\/doi.org\/10.1007\/978-3-642-41338-4_17","DOI":"10.1007\/978-3-642-41338-4_17"},{"key":"49_CR2","doi-asserted-by":"crossref","unstructured":"Das, R., Vukovic, M.: Emerging theories and models of human computation systems: a brief survey. In: Proceedings of the 2nd International Workshop on Ubiquitous Crowdsourcing, pp. 1\u20134 (2011)","DOI":"10.1145\/2030100.2030102"},{"key":"49_CR3","doi-asserted-by":"crossref","unstructured":"Demartini, G., Difallah, D.E., Cudr\u00e9-Mauroux, P.: Zencrowd: leveraging probabilistic reasoning and crowdsourcing techniques for large-scale entity linking. In: Proceedings of the 21st International Conference on World Wide Web, pp. 469\u2013478 (2012)","DOI":"10.1145\/2187836.2187900"},{"key":"49_CR4","doi-asserted-by":"crossref","unstructured":"Erez, E.S., Zhitomirsky-Geffet, M., Bar-Ilan, J.: Subjective vs. objective evaluation of ontological statements with crowdsourcing. In: Proceedings of the Association for Information Science and Technology, pp. 1\u20134 (2015)","DOI":"10.1002\/pra2.2015.145052010068"},{"issue":"12","key":"49_CR5","doi-asserted-by":"publisher","first-page":"4563","DOI":"10.3390\/su10124563","volume":"10","author":"A Giret","year":"2018","unstructured":"Giret, A., Carrascosa, C., Julian, V., Rebollo, M., Botti, V.: A crowdsourcing approach for sustainable last mile delivery. Sustainability 10(12), 4563 (2018)","journal-title":"Sustainability"},{"key":"49_CR6","doi-asserted-by":"publisher","unstructured":"Hertling, S., Paulheim, H.: WebIsALOD: providing hypernymy relations extracted from the web as linked open data. In: d\u2019Amato, C., et al. (eds.) ISWC 2017. LNCS, vol. 10588, pp. 111\u2013119. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-68204-4_11","DOI":"10.1007\/978-3-319-68204-4_11"},{"key":"49_CR7","doi-asserted-by":"crossref","unstructured":"Hevner, A.R., March, S.T., Park, J., Ram, S.: Design science in information systems research. In: MIS Quarterly, pp. 75\u2013105 (2004)","DOI":"10.2307\/25148625"},{"key":"49_CR8","doi-asserted-by":"crossref","unstructured":"Hogan, A., et al.: Knowledge graphs. ACM Comput. Surv. 54(4), 1\u201337 (2021)","DOI":"10.1145\/3447772"},{"key":"49_CR9","doi-asserted-by":"publisher","unstructured":"Iyer, V., Sanagavarapu, L.M., Raghu Reddy, Y.: A framework for syntactic and semantic quality evaluation of ontologies. In: International Conference on Secure Knowledge Management in Artificial Intelligence Era, pp. 73\u201393. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-97532-6_5","DOI":"10.1007\/978-3-030-97532-6_5"},{"key":"49_CR10","doi-asserted-by":"crossref","unstructured":"Jiang, J., et al.: Understanding crowdsourcing systems from a multiagent perspective and approach. ACM Trans. Autonom. Adapt. Syst. 13(2), 1\u201332 (2018)","DOI":"10.1145\/3226028"},{"key":"49_CR11","doi-asserted-by":"crossref","unstructured":"Kiptoo, C.C.: Ontology enhancement using crowdsourcing: a conceptual architecture. Int. J. Crowd Sci. (2020)","DOI":"10.1108\/IJCS-10-2019-0028"},{"key":"49_CR12","first-page":"265","volume":"394","author":"D Kontokostas","year":"2013","unstructured":"Kontokostas, D., Zaveri, A., Auer, S., Lehmann, J.: Triplecheckmate: a tool for crowdsourcing the quality assessment of linked data. Commun. Comput. Inf. Sci. 394, 265\u2013272 (2013)","journal-title":"Commun. Comput. Inf. Sci."},{"key":"49_CR13","unstructured":"Mao, A., Parkes, D.C., Procaccia, A.D., Zhang, H.: Human computation and multiagent systems: an algorithmic perspective. In: Proceedings of the 25th AAAI Conference on Artificial Intelligence. Citeseer (2011)"},{"issue":"4","key":"49_CR14","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3329124","volume":"52","author":"M McDaniel","year":"2019","unstructured":"McDaniel, M., Storey, V.C.: Evaluating domain ontologies: clarification, classification, and challenges. ACM Comp. Surv. 52(4), 1\u201344 (2019)","journal-title":"ACM Comp. Surv."},{"key":"49_CR15","doi-asserted-by":"crossref","unstructured":"Mortensen, J.M., et al.: Using the wisdom of the crowds to find critical errors in biomedical ontologies: a study of SNOMED CT. J. Am. Med. Inf. Assoc. 22(3), 640\u2013648 (2015)","DOI":"10.1136\/amiajnl-2014-002901"},{"key":"49_CR16","doi-asserted-by":"crossref","unstructured":"Mortensen, J.M., et al.: Is the crowd better as an assistant or a replacement in ontology engineering? An exploration through the lens of the gene ontology. J. Biomed. Inf. 60, 199\u2013209 (2016)","DOI":"10.1016\/j.jbi.2016.02.005"},{"key":"49_CR17","doi-asserted-by":"crossref","unstructured":"Nishioka, C., Scherp, A.: Analysing the evolution of knowledge graphs for the purpose of change verification. In: IEEE 12th International Conference on Semantic Computing, pp. 25\u201332. IEEE (2018)","DOI":"10.1109\/ICSC.2018.00013"},{"key":"49_CR18","doi-asserted-by":"publisher","unstructured":"Osborne, F., Motta, E.: Klink-2: integrating multiple web sources to generate semantic topic networks. In: Arenas, M., et al. (eds.) ISWC 2015. LNCS, vol. 9366, pp. 408\u2013424. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-25007-6_24","DOI":"10.1007\/978-3-319-25007-6_24"},{"key":"49_CR19","doi-asserted-by":"publisher","unstructured":"Osborne, F., Scavo, G., Motta, E.: Identifying diachronic topic-based research communities by clustering shared research trajectories. In: Presutti, V., d\u2019Amato, C., Gandon, F., d\u2019Aquin, M., Staab, S., Tordai, A. (eds.) ESWC 2014. LNCS, vol. 8465, pp. 114\u2013129. Springer, Cham (2014). https:\/\/doi.org\/10.1007\/978-3-319-07443-6_9","DOI":"10.1007\/978-3-319-07443-6_9"},{"issue":"3","key":"49_CR20","doi-asserted-by":"publisher","first-page":"489","DOI":"10.3233\/SW-160218","volume":"8","author":"H Paulheim","year":"2017","unstructured":"Paulheim, H.: Knowledge graph refinement: a survey of approaches and evaluation methods. Semantic Web J. 8(3), 489\u2013508 (2017)","journal-title":"Semantic Web J."},{"key":"49_CR21","doi-asserted-by":"crossref","unstructured":"Paulheim, H., Bizer, C.: Improving the quality of linked data using statistical distributions. Int. J. Semantic Web Inf. Syst. 10(2), 63\u201386 (2014)","DOI":"10.4018\/ijswis.2014040104"},{"key":"49_CR22","doi-asserted-by":"crossref","unstructured":"Pittet, P., Barth\u00e9l\u00e9my, J.: Exploiting users\u2019 feedbacks: towards a task-based evaluation of application ontologies throughout their lifecycle. In: International Conference on Knowledge Engineering and Ontology Development, vol. 2 (2015)","DOI":"10.5220\/0005608202630268"},{"key":"49_CR23","doi-asserted-by":"crossref","unstructured":"Poveda-Villal\u00f3n, M., G\u00f3mez-P\u00e9rez, A., Su\u00e1rez-Figueroa, M.C.: Oops!(ontology pitfall scanner!): an on-line tool for ontology evaluation. Int. J. Semantic Web Inf. Syst. 10(2), 7\u201334 (2014)","DOI":"10.4018\/ijswis.2014040102"},{"issue":"3","key":"49_CR24","doi-asserted-by":"publisher","first-page":"291","DOI":"10.3233\/SW-180292","volume":"9","author":"M Sabou","year":"2018","unstructured":"Sabou, M., Aroyo, L., Bontcheva, K., Bozzon, A., Qarout, R.K.: Semantic web and human computation: the status of an emerging field. Semantic Web J. 9(3), 291\u2013302 (2018)","journal-title":"Semantic Web J."},{"key":"49_CR25","unstructured":"Sabou, M., Fernandez, M., Poveda-Villal\u00f3n, M., Su\u00e1rez-Figueroa, M.C., Tsaneva, S.: Human-centric evaluation of semantic resources: a systematic mapping study. In preparation"},{"key":"49_CR26","doi-asserted-by":"crossref","unstructured":"Salatino, A.A., Osborne, F., Motta, E.: Augur: forecasting the emergence of new research topics. In: Proceedings of the 18th ACM\/IEEE on Joint Conference on Digital Libraries, pp. 303\u2013312 (2018)","DOI":"10.1145\/3197026.3197052"},{"key":"49_CR27","doi-asserted-by":"publisher","unstructured":"Sarasua, C., Simperl, E., Noy, N.F.: CrowdMap: crowdsourcing ontology alignment with microtasks. In: Cudr\u00e9-Mauroux, P., et al. (eds.) ISWC 2012. LNCS, vol. 7649, pp. 525\u2013541. Springer, Heidelberg (2012). https:\/\/doi.org\/10.1007\/978-3-642-35176-1_33","DOI":"10.1007\/978-3-642-35176-1_33"},{"key":"49_CR28","doi-asserted-by":"publisher","unstructured":"Tsaneva, S., K\u00e4sznar, K., Sabou, M.: Human-centric ontology evaluation: process and tool support. In: International Conference on Knowledge Engineering and Knowledge Management, pp. 182\u2013197. Springer, Cham (2022). https:\/\/doi.org\/10.1007\/978-3-031-17105-5_14","DOI":"10.1007\/978-3-031-17105-5_14"},{"key":"49_CR29","unstructured":"Tsaneva, S., Sabou, M.: A human computation approach for ontology restrictions verification. In: AAAI Conference on Human Computation and Crowdsourcing (2021). www.humancomputation.com\/2021\/assets\/wips_demos\/HCOMP_2021_paper_90.pdf"},{"issue":"4","key":"49_CR30","doi-asserted-by":"publisher","first-page":"379","DOI":"10.3233\/SW-150181","volume":"7","author":"G Wohlgenannt","year":"2016","unstructured":"Wohlgenannt, G., Sabou, M., Hanika, F.: Crowd-based ontology engineering with the ucomp prot\u00e9g\u00e9 plugin. Semantic Web 7(4), 379\u2013398 (2016)","journal-title":"Semantic Web"},{"key":"49_CR31","doi-asserted-by":"publisher","unstructured":"Wohlin, C., Runeson, P., H\u00f6st, M., Ohlsson, M.C., Regnell, B., Wessl\u00e9n, A.: Experimentation in Software Engineering. Springer, Heidelberg (2012). https:\/\/doi.org\/10.1007\/978-3-642-29044-2","DOI":"10.1007\/978-3-642-29044-2"}],"container-title":["Lecture Notes in Computer Science","The Semantic Web: ESWC 2023 Satellite Events"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-43458-7_49","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,3]],"date-time":"2024-01-03T14:06:32Z","timestamp":1704290792000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-43458-7_49"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031434570","9783031434587"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-43458-7_49","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"21 October 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ESWC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Semantic Web Conference","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Hersonissos","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Greece","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 May 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 June 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"20","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"esws2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2023.eswc-conferences.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Easy Chair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"109","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"50","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"46% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Submissions: Posters & Demos: 71 PhD Symposium: 24 Industry:14","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}