{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T10:18:56Z","timestamp":1777889936590,"version":"3.51.4"},"reference-count":40,"publisher":"SAGE Publications","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["SW"],"published-print":{"date-parts":[[2019,1,21]]},"DOI":"10.3233\/sw-180312","type":"journal-article","created":{"date-parts":[[2018,8,10]],"date-time":"2018-08-10T11:02:19Z","timestamp":1533898939000},"page":"293-304","source":"Crossref","is-referenced-by-count":19,"title":["Benchmarking question answering systems"],"prefix":"10.1177","volume":"10","author":[{"given":"Ricardo","family":"Usbeck","sequence":"first","affiliation":[{"name":"DICE \u2013 Data Science Group, Paderborn University, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"R\u00f6der","sequence":"additional","affiliation":[{"name":"DICE \u2013 Data Science Group, Paderborn University, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Hoffmann","sequence":"additional","affiliation":[{"name":"AKSW Group, University of Leipzig, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Felix","family":"Conrads","sequence":"additional","affiliation":[{"name":"DICE \u2013 Data Science Group, Paderborn University, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jonathan","family":"Huthmann","sequence":"additional","affiliation":[{"name":"AKSW Group, University of Leipzig, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Axel-Cyrille","family":"Ngonga-Ngomo","sequence":"additional","affiliation":[{"name":"DICE \u2013 Data Science Group, Paderborn University, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christian","family":"Demmler","sequence":"additional","affiliation":[{"name":"AKSW Group, University of Leipzig, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christina","family":"Unger","sequence":"additional","affiliation":[{"name":"CITEC, University of Bielefeld, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"179","reference":[{"key":"10.3233\/SW-180312_ref1","unstructured":"M.\u00a0Agosti, G.M.D.\u00a0Nunzio, M.\u00a0Dussin and N.\u00a0Ferro, 10 years of CLEF data in DIRECT: where we are and where we can go, in: Proceedings of the 3rd International Workshop on Evaluating Information Access, EVIA 2010, National Center of Sciences, Tokyo, Japan, June 15, 2010, T.\u00a0Sakai, M.\u00a0Sanderson and W.\u00a0Webber, eds, National Institute of Informatics (NII), pp.\u00a016\u201324. http:\/\/research.nii.ac.jp\/ntcir\/workshop\/OnlineProceedings8\/EVIA\/04-EVIA2010-AgostiM.pdf. ISBN 978-4-86049-054-6."},{"key":"10.3233\/SW-180312_ref3","doi-asserted-by":"crossref","unstructured":"P.\u00a0Baudi\u0161 and J.\u00a0\u0160ediv\u00fd, Modeling of the question answering task in the YodaQA system, in: CLEF\u201915, Springer International Publishing, 2015, pp.\u00a0222\u2013228.","DOI":"10.1007\/978-3-319-24027-5_20"},{"key":"10.3233\/SW-180312_ref4","unstructured":"R.\u00a0Beaumont, B.\u00a0Grau and A.-L.\u00a0Ligozat, Semgraphqa at qald-5: Limsi participation at qald-5 at clef, in: CLEF (Working Notes), 2015."},{"key":"10.3233\/SW-180312_ref5","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-34129-3_38"},{"key":"10.3233\/SW-180312_ref6","doi-asserted-by":"crossref","unstructured":"M.\u00a0Br\u00fcmmer, C.\u00a0Baron, I.\u00a0Ermilov, M.\u00a0Freudenberg, D.\u00a0Kontokostas and S.\u00a0Hellmann, DataID: Towards semantically rich metadata for complex datasets, in: 10th International Conference on Semantic Systems 2014, 2014.","DOI":"10.1145\/2660517.2660538"},{"key":"10.3233\/SW-180312_ref7","doi-asserted-by":"crossref","unstructured":"E.\u00a0Cabrio, J.\u00a0Cojan, F.\u00a0Gandon and A.\u00a0Hallili, Querying multilingual DBpedia with QAKiS, in: ESWC, 2013, pp.\u00a0194\u2013198.","DOI":"10.1007\/978-3-642-41242-4_23"},{"key":"10.3233\/SW-180312_ref8","doi-asserted-by":"crossref","unstructured":"C.\u00a0Comparot, O.\u00a0Haemmerl\u00e9 and N.\u00a0Hernandez, An easy way of expressing conceptual graph queries from keywords and query patterns, in: International Conference on Conceptual Structures, Springer, 2010, pp.\u00a084\u201396.","DOI":"10.1007\/978-3-642-14197-3_11"},{"key":"10.3233\/SW-180312_ref10","doi-asserted-by":"crossref","unstructured":"D.\u00a0Damljanovic, M.\u00a0Agatonovic and H.\u00a0Cunningham, Natural language interfaces to ontologies: Combining syntactic analysis and ontology-based lookup through the user interaction, in: Extended Semantic Web Conference, Springer, 2010, pp.\u00a0106\u2013120.","DOI":"10.1007\/978-3-642-13486-9_8"},{"key":"10.3233\/SW-180312_ref11","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-017-1100-y"},{"key":"10.3233\/SW-180312_ref12","unstructured":"D.\u00a0Diefenbach, T.P.\u00a0Tanon, K.\u00a0Singh and P.\u00a0Maret, Question answering benchmarks for wikidata, in: ISWC 2017, Vienne, Austria, 2017, https:\/\/hal.archives-ouvertes.fr\/hal-01637141."},{"key":"10.3233\/SW-180312_ref13","unstructured":"C.\u00a0Dima, Intui2: A prototype system for question answering over linked data, in: CLEF (Working Notes), 2013."},{"key":"10.3233\/SW-180312_ref14","unstructured":"C.\u00a0Dima, Answering natural language questions with intui3, in: CLEF (Working Notes), 2014, pp.\u00a01201\u20131211."},{"key":"10.3233\/SW-180312_ref15","unstructured":"S.\u00a0Ferr\u00e9, squall2sparql: A translator from controlled english to full sparql 1.1, in: Work. Multilingual Question Answering over Linked Data (QALD-3), 2013."},{"key":"10.3233\/SW-180312_ref16","unstructured":"C.\u00a0Giannone, V.\u00a0Bellomaria and R.\u00a0Basili, A hmm-based approach to question answering against linked data, in: CLEF (Working Notes), Citeseer, 2013."},{"key":"10.3233\/SW-180312_ref17","doi-asserted-by":"crossref","unstructured":"B.F.\u00a0Green Jr., A.K.\u00a0Wolf, C.\u00a0Chomsky and K.\u00a0Laughery, Baseball: An automatic question-answerer, in: Western Joint IRE-AIEE-ACM Computer Conference, May 9\u201311, 1961, ACM, 1961, pp.\u00a0219\u2013224.","DOI":"10.1145\/1460690.1460714"},{"key":"10.3233\/SW-180312_ref18","doi-asserted-by":"crossref","unstructured":"S.\u00a0He, S.\u00a0Liu, Y.\u00a0Chen, G.\u00a0Zhou, K.\u00a0Liu and J.\u00a0Zhao, Casia@ qald-3: A question answering system over linked data, in: CLEF (Working Notes), 2013.","DOI":"10.3115\/v1\/D14-1116"},{"key":"10.3233\/SW-180312_ref19","doi-asserted-by":"crossref","unstructured":"S.\u00a0He, Y.\u00a0Zhang, K.\u00a0Liu and J.\u00a0Zhao, Casia@ v2: A mln-based question answering system over linked data, in: CLEF (Working Notes), 2014, pp.\u00a01249\u20131259.","DOI":"10.3115\/v1\/D14-1116"},{"key":"10.3233\/SW-180312_ref21","unstructured":"G.\u00a0Joris and S.\u00a0Ferr\u00e9, Scalewelis: A scalable query-based faceted search system on top of sparql endpoints, in: Work. Multilingual Question Answering over Linked Data (QALD-3), 2013."},{"key":"10.3233\/SW-180312_ref22","unstructured":"J.\u00a0Kim, G.\u00a0Choi, J.\u00a0Kim, E.\u00a0Kim and K.\u00a0Choi, The open framework for developing knowledge base and question answering system, in: COLING 2016, 26th International Conference on Computational Linguistics, Proceedings of the Conference System Demonstrations, Osaka, Japan, December 11\u201316, 2016, H.\u00a0Watanabe, ed., ACL, 2016, pp.\u00a0161\u2013165. http:\/\/aclweb.org\/anthology\/C\/C16\/C16-2034.pdf. ISBN 978-4-87974-703-7."},{"key":"10.3233\/SW-180312_ref24","doi-asserted-by":"crossref","unstructured":"V.\u00a0Lopez, E.\u00a0Motta and V.\u00a0Uren, Poweraqua: Fishing the semantic web, in: European Semantic Web Conference, Springer, 2006, pp.\u00a0393\u2013410.","DOI":"10.1007\/11762256_30"},{"key":"10.3233\/SW-180312_ref28","unstructured":"S.\u00a0Park, S.\u00a0Kwon, B.\u00a0Kim and G.G.\u00a0Lee, Isoft at qald-5: Hybrid question answering system over linked data and text data, in: CLEF (Working Notes), 2015."},{"key":"10.3233\/SW-180312_ref29","unstructured":"C.\u00a0Pradel, G.\u00a0Peyet, O.\u00a0Haemmerl\u00e9 and N.\u00a0Hernandez, Swip at qald-3: Results, criticisms and lesson learned, in: Information Access Evaluation. Multilinguality, Multimodality, and Visualization. PROMISE Network of Excellence, 2013."},{"key":"10.3233\/SW-180312_ref32","unstructured":"S.\u00a0Ruseti, A.\u00a0Mirea, T.\u00a0Rebedea and S.\u00a0Trausan-Matu, Qanswer-enhanced entity matching for question answering over linked data, in: CLEF (Working Notes), 2015."},{"key":"10.3233\/SW-180312_ref33","unstructured":"M.\u00a0Saleem, S.N.\u00a0Dastjerdi, R.\u00a0Usbeck and A.N.\u00a0Ngomo, Question answering over linked data: What is difficult to answer? What affects the F scores? in: Joint Proceedings of BLINK2017: 2nd International Workshop on Benchmarking Linked Data and NLIWoD3: Natural Language Interfaces for the Web of Data Co-Located with 16th International Semantic Web Conference (ISWC 2017), Vienna, Austria, October 21\u201322, 2017, R.\u00a0Usbeck, A.N.\u00a0Ngomo, J.\u00a0Kim, K.\u00a0Choi, P.\u00a0Cimiano, I.\u00a0Fundulaki and A.\u00a0Krithara, eds, CEUR Workshop Proceedings, Vol.\u00a01932, CEUR-WS.org, 2017, http:\/\/ceur-ws.org\/Vol-1932\/paper-02.pdf."},{"key":"10.3233\/SW-180312_ref35","doi-asserted-by":"crossref","unstructured":"K.\u00a0Singh, A.\u00a0Both, D.\u00a0Diefenbach, S.\u00a0Shekarpour, D.\u00a0Cherix and C.\u00a0Lange, Qanary \u2013 the fast track to creating a question answering system with linked data technology, in: ESWC, 2016.","DOI":"10.1007\/978-3-319-47602-5_36"},{"key":"10.3233\/SW-180312_ref36","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-68204-4_22"},{"key":"10.3233\/SW-180312_ref37","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1186\/s12859-015-0564-6","article-title":"An overview of the BIOASQ large-scale biomedical semantic indexing and question answering competition","volume":"16","author":"Tsatsaronis","year":"2015","journal-title":"BMC Bioinformatics"},{"key":"10.3233\/SW-180312_ref38","unstructured":"C.\u00a0Unger, C.\u00a0Forascu, V.\u00a0Lopez, A.N.\u00a0Ngomo, E.\u00a0Cabrio, P.\u00a0Cimiano and S.\u00a0Walter, Question answering over linked data (QALD-4), in: CLEF, 2014, pp.\u00a01172\u20131180."},{"key":"10.3233\/SW-180312_ref39","unstructured":"C.\u00a0Unger, C.\u00a0Forascu, V.\u00a0Lopez, A.N.\u00a0Ngomo, E.\u00a0Cabrio, P.\u00a0Cimiano and S.\u00a0Walter, Question answering over linked data (QALD-5), in: CLEF, 2015, http:\/\/ceur-ws.org\/Vol-1391\/173-CR.pdf."},{"key":"10.3233\/SW-180312_ref40","doi-asserted-by":"crossref","unstructured":"C.\u00a0Unger, A.-C.N.\u00a0Ngomo and E.\u00a0Cabrio, in: 6th Open Challenge on Question Answering over Linked Data (QALD-6), Springer International Publishing, Cham, 2016, pp.\u00a0171\u2013177. ISBN 978-3-319-46565-4.","DOI":"10.1007\/978-3-319-46565-4_13"},{"key":"10.3233\/SW-180312_ref41","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-69146-6_6"},{"key":"10.3233\/SW-180312_ref42","doi-asserted-by":"crossref","unstructured":"R.\u00a0Usbeck, A.N.\u00a0Ngomo, L.\u00a0B\u00fchmann and C.\u00a0Unger, HAWK \u2013 hybrid question answering using linked data, in: Proceedings, the Semantic Web. Latest Advances and New Domains \u2013 12th European Semantic Web Conference, ESWC 2015, Portoroz, Slovenia, May 31\u2013June 4, 2015. 2015, pp.\u00a0353\u2013368.","DOI":"10.1007\/978-3-319-18818-8_22"},{"key":"10.3233\/SW-180312_ref43","unstructured":"R.\u00a0Usbeck and A.-C.\u00a0Ngonga Ngomo, HAWK@QALD5\u00a0\u2013 trying to answer hybrid questions with various simple ranking techniques, in: CLEF 2015 Labs and Workshops, Notebook Papers, CEUR Workshop Proceedings, Vo.\u00a01391, CEUR-WS.org, 2015, http:\/\/svn.aksw.org\/papers\/2015\/CLEF_HAWK\/public.pdf."},{"key":"10.3233\/SW-180312_ref44","doi-asserted-by":"crossref","unstructured":"R.\u00a0Usbeck, M.\u00a0R\u00f6der, P.\u00a0Haase, A.\u00a0Kozlov, M.\u00a0Saleem and A.-C.N.\u00a0Ngomo, Requirements to modern semantic search engines, in: KESW, 2016.","DOI":"10.1007\/978-3-319-45880-9_25"},{"key":"10.3233\/SW-180312_ref45","unstructured":"R.\u00a0Usbeck, M.\u00a0R\u00f6der, A.-C.\u00a0Ngonga Ngomo, C.\u00a0Baron, A.\u00a0Both, M.\u00a0Br\u00fcmmer, D.\u00a0Ceccarelli, M.\u00a0Cornolti, D.\u00a0Cherix, B.\u00a0Eickmann, P.\u00a0Ferragina, C.\u00a0Lemke, A.\u00a0Moro, R.\u00a0Navigli, F.\u00a0Piccinno, G.\u00a0Rizzo, H.\u00a0Sack, R.\u00a0Speck, R.\u00a0Troncy, J.\u00a0Waitelonis and L.\u00a0Wesemann, GERBIL \u2013 general entity annotation benchmark framework, in: 24th WWW Conference, 2015."},{"key":"10.3233\/SW-180312_ref46","doi-asserted-by":"crossref","unstructured":"A.P.B.\u00a0Veyseh, Cross-lingual question answering using common semantic space, in: Proceedings of TextGraphs@NAACL-HLT 2016: The 10th Workshop on Graph-Based Methods for Natural Language Processing, San Diego, California, USA, June 17, 2016, 2016, pp.\u00a015\u201319. http:\/\/aclweb.org\/anthology\/W\/W16\/W16-1403.pdf.","DOI":"10.18653\/v1\/W16-1403"},{"key":"10.3233\/SW-180312_ref47","doi-asserted-by":"crossref","unstructured":"E.M.\u00a0Voorhees et al., The trec-8 question answering track report, in: Trec, Vol.\u00a099, 1999, pp.\u00a077\u201382.","DOI":"10.6028\/NIST.SP.500-246.qa-overview"},{"key":"10.3233\/SW-180312_ref48","doi-asserted-by":"crossref","unstructured":"M.\u00a0Wendt, M.\u00a0Gerlach and H.\u00a0D\u00fcwiger, Linguistic modeling of linked open data for question answering, in: Extended Semantic Web Conference, Springer, 2012, pp.\u00a0102\u2013116.","DOI":"10.1007\/978-3-662-46641-4_8"},{"key":"10.3233\/SW-180312_ref50","doi-asserted-by":"crossref","unstructured":"K.\u00a0Xu, Y.\u00a0Feng and D.\u00a0Zhao, Xser@ QALD-4: Answering Natural Language Questions Via Phrasal Semantic Parsing. QALD-4, 2014.","DOI":"10.1007\/978-3-662-45924-9_30"},{"key":"10.3233\/SW-180312_ref51","doi-asserted-by":"crossref","unstructured":"L.\u00a0Zou, R.\u00a0Huang, H.\u00a0Wang, J.X.\u00a0Yu, W.\u00a0He and D.\u00a0Zhao, Natural language question answering over rdf: A graph data driven approach, in: Proceedings of the 2014 ACM SIGMOD International Conference on Management of Data, ACM, 2014, pp.\u00a0313\u2013324.","DOI":"10.1145\/2588555.2610525"}],"container-title":["Semantic Web"],"original-title":[],"link":[{"URL":"https:\/\/content.iospress.com\/download?id=10.3233\/SW-180312","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T05:24:06Z","timestamp":1777613046000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/full\/10.3233\/SW-180312"}},"subtitle":[],"editor":[{"given":"Ruben","family":"Verborgh","sequence":"additional","affiliation":[{"name":"Ghent University \u2013 iMinds, Belgium"}],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Axel-Cyrille","family":"Ngonga Ngomo","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Irini","family":"Fundulaki","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Anastasia","family":"Krithara","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]}],"short-title":[],"issued":{"date-parts":[[2019,1,21]]},"references-count":40,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.3233\/sw-180312","relation":{},"ISSN":["2210-4968","1570-0844"],"issn-type":[{"value":"2210-4968","type":"electronic"},{"value":"1570-0844","type":"print"}],"subject":[],"published":{"date-parts":[[2019,1,21]]}}}