{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T10:18:09Z","timestamp":1777889889434,"version":"3.51.4"},"reference-count":35,"publisher":"SAGE Publications","issue":"4","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["SW"],"published-print":{"date-parts":[[2016,5,27]]},"DOI":"10.3233\/sw-150181","type":"journal-article","created":{"date-parts":[[2016,5,27]],"date-time":"2016-05-27T07:48:48Z","timestamp":1464335328000},"page":"379-398","source":"Crossref","is-referenced-by-count":14,"title":["Crowd-based ontology engineering with\u00a0the\u00a0uComp Prot\u00e9g\u00e9 plugin"],"prefix":"10.1177","volume":"7","author":[{"given":"Gerhard","family":"Wohlgenannt","sequence":"first","affiliation":[{"name":"Vienna University of Economics and Business (WU), Welthandelsplatz 1, 1020 Vienna, Austria. E-mails:\u00a0gerhard.wohlgenannt@wu.ac.at,\u00a0florian.hanika@gmail.com"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Marta","family":"Sabou","sequence":"additional","affiliation":[{"name":"Vienna University of Technology (TUW), Inst. of Software Technology, CDL-Flex, Favoritenstrasse 9-11\/188, 1040 Vienna, Austria. E-mail:\u00a0marta.sabou@ifs.tuwien.ac.at"},{"name":"MODUL University Vienna, Am Kahlneberg 1, 1190 Vienna, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Florian","family":"Hanika","sequence":"additional","affiliation":[{"name":"Vienna University of Economics and Business (WU), Welthandelsplatz 1, 1020 Vienna, Austria. E-mails:\u00a0gerhard.wohlgenannt@wu.ac.at,\u00a0florian.hanika@gmail.com"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"179","reference":[{"issue":"4","key":"10.3233\/SW-150181_ref1","doi-asserted-by":"crossref","first-page":"1007","DOI":"10.1007\/s10579-013-9215-6","article-title":"GATE teamware: A web-based, collaborative text annotation framework","volume":"47","author":"Bontcheva","year":"2013","journal-title":"Lang. Resour. Eval."},{"key":"10.3233\/SW-150181_ref2","doi-asserted-by":"crossref","unstructured":"K.\u00a0Bontcheva, I.\u00a0Roberts, L.\u00a0Derczynski and D.P.\u00a0Rout, The GATE crowdsourcing plugin: Crowdsourcing annotated corpora made easy, in: Proc. of the 14th Conference of the European Chapter of the Association for Computational Linguistics, EACL 2014, April 26\u201330, 2014, Gothenburg, Sweden, The Association for Computer Linguistics, 2014, pp.\u00a097\u2013100.","DOI":"10.3115\/v1\/E14-2025"},{"key":"10.3233\/SW-150181_ref3","first-page":"194","article-title":"SUS-a quick and dirty usability scale","volume":"189","author":"Brooke","year":"1996","journal-title":"Usability Evaluation in Industry"},{"key":"10.3233\/SW-150181_ref4","doi-asserted-by":"crossref","unstructured":"I.\u00a0Celino, S.\u00a0Contessa, M.\u00a0Corubolo, D.\u00a0Dell\u2019Aglio, E.\u00a0Della\u00a0Valle, S.\u00a0Fumeo and T.\u00a0Kruger, Linking smart cities datasets with human computation \u2013 the case of UrbanMatch, in: Proc. of the International Semantic Web Conference, 2012, pp.\u00a034\u201349.","DOI":"10.1007\/978-3-642-35173-0_3"},{"key":"10.3233\/SW-150181_ref5","unstructured":"M.\u00a0Cheatham and P.\u00a0Hitzler, Conference v2.0: An uncertain version of the OAEI conference benchmark, in: The Semantic Web \u2013 ISWC 2014, P.\u00a0Mika, T.\u00a0Tudorache, A.\u00a0Bernstein, C.\u00a0Welty, C.\u00a0Knoblock, D.\u00a0Vrandecic, P.\u00a0Groth, N.\u00a0Noy, K.\u00a0Janowicz and C.\u00a0Goble, eds, LNCS, Vol.\u00a08797, Springer International Publishing, 2014, pp.\u00a033\u201348."},{"key":"10.3233\/SW-150181_ref6","doi-asserted-by":"crossref","unstructured":"L.B.\u00a0Chilton, G.\u00a0Little, D.\u00a0Edge, D.S.\u00a0Weld and J.A.\u00a0Landay, Cascade: Crowdsourcing taxonomy creation, in: Proc. of the SIGCHI Conference on Human Factors in Computing Systems, CHI \u201913, ACM, 2013, pp.\u00a01999\u20132008.","DOI":"10.1145\/2470654.2466265"},{"issue":"7307","key":"10.3233\/SW-150181_ref7","doi-asserted-by":"crossref","first-page":"756","DOI":"10.1038\/nature09304","article-title":"Predicting protein structures with a multiplayer online game","volume":"466","author":"Cooper","year":"2010","journal-title":"Nature"},{"key":"10.3233\/SW-150181_ref8","doi-asserted-by":"crossref","unstructured":"G.\u00a0Demartini, D.E.\u00a0Difallah and P.\u00a0Cudr\u00e9-Mauroux, ZenCrowd: Leveraging probabilistic reasoning and crowdsourcing techniques for large-scale entity linking, in: Proc. of the 21st Int. Conf. on World Wide Web, ACM, 2012, pp.\u00a0469\u2013478.","DOI":"10.1145\/2187836.2187900"},{"key":"10.3233\/SW-150181_ref9","doi-asserted-by":"crossref","unstructured":"K.\u00a0Eckert, M.\u00a0Niepert, C.\u00a0Niemann, C.\u00a0Buckner, C.\u00a0Allen and H.\u00a0Stuckenschmidt, Crowdsourcing the assembly of concept hierarchies, in: Proc. of the 10th Annual Joint Conf. on Digital Libraries, JCDL \u201910, ACM, 2010, pp.\u00a0139\u2013148.","DOI":"10.1145\/1816123.1816143"},{"issue":"2","key":"10.3233\/SW-150181_ref10","doi-asserted-by":"crossref","first-page":"413","DOI":"10.1162\/COLI_a_00057","article-title":"Amazon mechanical Turk: Gold mine or coal mine?","volume":"37","author":"Fort","year":"2011","journal-title":"Computational Linguistics"},{"issue":"1","key":"10.3233\/SW-150181_ref11","doi-asserted-by":"crossref","first-page":"45","DOI":"10.1017\/S0269888910000408","article-title":"Interactive knowledge capture in the new millennium: How the Semantic Web changed everything","volume":"26","author":"Gil","year":"2011","journal-title":"The Knowledge Engineering Review"},{"key":"10.3233\/SW-150181_ref13","doi-asserted-by":"crossref","unstructured":"A.\u00a0Kawrykow, G.\u00a0Roumanis, A.\u00a0Kam, D.\u00a0Kwak, C.\u00a0Leung, C.\u00a0Wu, E.\u00a0Zarour and Phylo players, Phylo: A citizen science approach for improving multiple sequence alignment, PLoS ONE 7(3) (2012), e31362.","DOI":"10.1371\/journal.pone.0031362"},{"key":"10.3233\/SW-150181_ref14","doi-asserted-by":"crossref","unstructured":"A.\u00a0Kittur, E.H.\u00a0Chi and B.\u00a0Suh, Crowdsourcing user studies with mechanical Turk, in: Proc. of Human Factors in Computing Systems, Florence, Italy, 2008, pp.\u00a0453\u2013456.","DOI":"10.1145\/1357054.1357127"},{"issue":"1","key":"10.3233\/SW-150181_ref15","doi-asserted-by":"crossref","first-page":"159","DOI":"10.2307\/2529310","article-title":"The measurement of observer agreement for categorical data","volume":"33","author":"Landis","year":"1977","journal-title":"Biometrics"},{"key":"10.3233\/SW-150181_ref16","unstructured":"F.\u00a0Laws, C.\u00a0Scheible and H.\u00a0Sch\u00fctze, Active learning with amazon mechanical Turk, in: Proc. of the Conference on Empirical Methods in Natural Language Processing, EMNLP \u201911, ACL, 2011, pp.\u00a01546\u20131556."},{"key":"10.3233\/SW-150181_ref17","unstructured":"H.\u00a0Lieberman, D.\u00a0Smith and A.\u00a0Teeters, Common consensus: A web-based game for collecting commonsense goals, in: Proc. of the Workshop on Common Sense and Intelligent User Interfaces Held in Conjunction with the 2007 International Conference on Intelligent User Interfaces (IUI 2007), Hawaii, USA, 2007."},{"key":"10.3233\/SW-150181_ref18","unstructured":"T.\u00a0Markotschi and J.\u00a0V\u00f6lker, Guess what?! Human intelligence for mining linked data, in: Proc. of the WS. on Knowledge Injection Into and Extraction from Linked Data at EKAW, Lisbon, Portugal, CEUR, October 15, 2010, pp.\u00a028\u201339."},{"key":"10.3233\/SW-150181_ref19","doi-asserted-by":"crossref","unstructured":"N.F.\u00a0Noy, J.\u00a0Mortensen, M.A.\u00a0Musen and P.R.\u00a0Alexander, Mechanical Turk as an ontology engineer?: Using microtasks as a component of an ontology-engineering workflow, in: Proc. of the 5th Annual ACM Web Science Conf., WebSci \u201913, 2013, pp.\u00a0262\u2013271.","DOI":"10.1145\/2464464.2464482"},{"issue":"1","key":"10.3233\/SW-150181_ref20","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/2448116.2448119","article-title":"Phrase detectives: Utilizing collective intelligence for Internet-scale language resource creation","volume":"3","author":"Poesio","year":"2013","journal-title":"Transactions on Interactive Intelligent Systems"},{"key":"10.3233\/SW-150181_ref21","doi-asserted-by":"crossref","unstructured":"D.\u00a0Retelny, S.\u00a0Robaszkiewicz, A.\u00a0To, W.S.\u00a0Lasecki, J.\u00a0Patel, N.\u00a0Rahmati, T.\u00a0Doshi, M.\u00a0Valentine and M.S.\u00a0Bernstein, Expert crowdsourcing with flash teams, in: Proc. of the 27th Annual ACM Symposium on User Interface Software and Technology, UIST \u201914, ACM, 2014, pp.\u00a075\u201385.","DOI":"10.1145\/2642918.2647409"},{"key":"10.3233\/SW-150181_ref22","doi-asserted-by":"crossref","unstructured":"M.\u00a0Sabou, K.\u00a0Bontcheva and A.\u00a0Scharl, Crowdsourcing research opportunities: Lessons from natural language processing, in: Proc. of the 12th International Conference on Knowledge Management and Knowledge Technologies, I-KNOW \u201912, ACM, 2012, pp.\u00a017:1\u201317:8.","DOI":"10.1145\/2362456.2362479"},{"key":"10.3233\/SW-150181_ref23","doi-asserted-by":"crossref","unstructured":"M.\u00a0Sabou, K.\u00a0Bontcheva, A.\u00a0Scharl and M.\u00a0F\u00f6ls, Games with a purpose or mechanised labour?: A comparative study, in: Proc. of the 13th Int. Conf. on Knowledge Management and Knowledge Technologies (iKNOW), 2013, pp.\u00a01\u20138.","DOI":"10.1145\/2494188.2494210"},{"issue":"3","key":"10.3233\/SW-150181_ref24","doi-asserted-by":"crossref","first-page":"14","DOI":"10.4018\/ijswis.2013070102","article-title":"Crowdsourced knowledge acquisition: Towards hybrid-genre workflows","volume":"9","author":"Sabou","year":"2013","journal-title":"Int. J. of Semantic Web and Information Systems"},{"key":"10.3233\/SW-150181_ref25","doi-asserted-by":"crossref","unstructured":"C.\u00a0Sarasua, E.\u00a0Simperl and N.F.\u00a0Noy, CrowdMap: Crowdsourcing ontology alignment with microtasks, in: Proc. of the 11th Int. Conf. on the Semantic Web, ISWC\u201912, 2012, pp.\u00a0525\u2013541.","DOI":"10.1007\/978-3-642-35176-1_33"},{"key":"10.3233\/SW-150181_ref26","doi-asserted-by":"crossref","unstructured":"A.\u00a0Scharl, M.\u00a0Sabou and M.\u00a0F\u00f6ls, Climate quiz: A web application for eliciting and validating knowledge from social networks, in: Proc. of the 18th Brazilian Symposium on Multimedia and the Web, WebMedia \u201912, ACM, 2012, pp.\u00a0189\u2013192.","DOI":"10.1145\/2382636.2382677"},{"issue":"3","key":"10.3233\/SW-150181_ref27","doi-asserted-by":"crossref","first-page":"50","DOI":"10.1109\/MIS.2008.45","article-title":"Games with a purpose for the Semantic Web","volume":"23","author":"Siorpaes","year":"2008","journal-title":"Intelligent Systems, IEEE"},{"key":"10.3233\/SW-150181_ref28","doi-asserted-by":"crossref","unstructured":"S.\u00a0Thaler, E.\u00a0Simperl and K.\u00a0Siorpaes, SpotTheLink: Playful alignment of ontologies, in: Proc. of the ACM Symp. on Applied Computing, 2011, pp.\u00a01711\u20131712.","DOI":"10.1145\/1982185.1982542"},{"issue":"5","key":"10.3233\/SW-150181_ref29","doi-asserted-by":"crossref","first-page":"52","DOI":"10.1109\/MIC.2012.67","article-title":"An experiment in comparing human-computation techniques","volume":"16","author":"Thaler","year":"2012","journal-title":"IEEE Internet Computing"},{"issue":"1","key":"10.3233\/SW-150181_ref30","doi-asserted-by":"crossref","first-page":"89","DOI":"10.3233\/SW-2012-0057","article-title":"WebProtege: A collaborative ontology editor and knowledge acquisition tool for the web","volume":"4","author":"Tudorache","year":"2013","journal-title":"Semant. Web J."},{"key":"10.3233\/SW-150181_ref31","doi-asserted-by":"crossref","unstructured":"D.\u00a0Vickrey, A.\u00a0Bronzan, W.\u00a0Choi, A.\u00a0Kumar, J.\u00a0Turner-Maier, A.\u00a0Wang and D.\u00a0Koller, Online word games for semantic data collection, in: Proc. of the Conference on Empirical Methods in Natural Language Processing, 2008, pp.\u00a0533\u2013542.","DOI":"10.3115\/1613715.1613781"},{"issue":"8","key":"10.3233\/SW-150181_ref32","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/1378704.1378719","article-title":"Designing games with a purpose","volume":"51","author":"von\u00a0Ahn","year":"2008","journal-title":"Commun. ACM"},{"key":"10.3233\/SW-150181_ref33","doi-asserted-by":"crossref","unstructured":"L.\u00a0von\u00a0Ahn, M.\u00a0Kedia and M.\u00a0Blum, Verbosity: A game for collecting common-sense facts, in: Proc. of the SIGCHI Conference on Human Factors in Computing Systems, CHI, ACM, New York, NY, USA, 2006, pp.\u00a075\u201378.","DOI":"10.1145\/1124772.1124784"},{"issue":"4","key":"10.3233\/SW-150181_ref34","doi-asserted-by":"crossref","first-page":"236","DOI":"10.1108\/17415651111189478","article-title":"WhoKnows? Evaluating linked data heuristics with a quiz that cleans up DBpedia","volume":"8","author":"Waitelonis","year":"2011","journal-title":"Interact. Techn. Smart Edu."},{"issue":"3","key":"10.3233\/SW-150181_ref35","first-page":"243","article-title":"Dynamic integration of multiple evidence sources for ontology learning","volume":"3","author":"Wohlgenannt","year":"2012","journal-title":"Journal of Information and Data Management"},{"key":"10.3233\/SW-150181_ref36","doi-asserted-by":"crossref","unstructured":"L.\u00a0Wolf, M.\u00a0Knuth, J.\u00a0Osterhoff and H.\u00a0Sack, RISQ! Renowned individuals semantic quiz \u2013 a jeopardy like quiz game for ranking facts, in: Proc. of the 7th Int. Conf. on Semantic Systems, I-Semantics \u201911, 2011, pp.\u00a071\u201378.","DOI":"10.1145\/2063518.2063528"}],"container-title":["Semantic Web"],"original-title":[],"link":[{"URL":"https:\/\/content.iospress.com\/download?id=10.3233\/SW-150181","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T05:22:58Z","timestamp":1777612978000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/full\/10.3233\/SW-150181"}},"subtitle":[],"editor":[{"given":"Krzysztof","family":"Janowicz","sequence":"additional","affiliation":[{"name":"STKO Lab, University of California, Santa Barbara, USA"}],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Stefan","family":"Schlobach","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, The Netherlands"}],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Stefan","family":"Schlobach","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Krzysztof","family":"Janowicz","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]}],"short-title":[],"issued":{"date-parts":[[2016,5,27]]},"references-count":35,"journal-issue":{"issue":"4"},"URL":"https:\/\/doi.org\/10.3233\/sw-150181","relation":{},"ISSN":["2210-4968","1570-0844"],"issn-type":[{"value":"2210-4968","type":"electronic"},{"value":"1570-0844","type":"print"}],"subject":[],"published":{"date-parts":[[2016,5,27]]}}}