{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,11]],"date-time":"2026-05-11T11:30:15Z","timestamp":1778499015860,"version":"3.51.4"},"publisher-location":"Cham","reference-count":36,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030307950","type":"print"},{"value":"9783030307967","type":"electronic"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-30796-7_18","type":"book-chapter","created":{"date-parts":[[2019,10,16]],"date-time":"2019-10-16T23:21:54Z","timestamp":1571268114000},"page":"277-292","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["QaldGen: Towards Microbenchmarking of Question Answering Systems over Knowledge Graphs"],"prefix":"10.1007","author":[{"given":"Kuldeep","family":"Singh","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Muhammad","family":"Saleem","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Abhishek","family":"Nadgeri","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Felix","family":"Conrads","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jeff Z.","family":"Pan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Axel-Cyrille Ngonga","family":"Ngomo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jens","family":"Lehmann","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2019,10,17]]},"reference":[{"key":"18_CR1","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"722","DOI":"10.1007\/978-3-540-76298-0_52","volume-title":"The Semantic Web","author":"S Auer","year":"2007","unstructured":"Auer, S., Bizer, C., Kobilarov, G., Lehmann, J., Cyganiak, R., Ives, Z.: DBpedia: a nucleus for a web of open data. In: Aberer, K., et al. (eds.) ASWC\/ISWC -2007. LNCS, vol. 4825, pp. 722\u2013735. Springer, Heidelberg (2007). https:\/\/doi.org\/10.1007\/978-3-540-76298-0_52"},{"key":"18_CR2","unstructured":"Berant, J., Chou, A., Frostig, R., Liang, P.: Semantic parsing on freebase from question-answer pairs. In: Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing, EMNLP 2013, pp. 1533\u20131544. ACL (2013)"},{"key":"18_CR3","doi-asserted-by":"crossref","unstructured":"Bollacker, K.D., Evans, C., Paritosh, P., Sturge, T., Taylor, J.: Freebase: a collaboratively created graph database for structuring human knowledge. In: ACM SIGMOD, pp. 1247\u20131250 (2008)","DOI":"10.1145\/1376616.1376746"},{"key":"18_CR4","doi-asserted-by":"crossref","unstructured":"Choi, K.-S., Mitamura, T., Vossen, P., Kim, J.-D., Ngomo, A.-C.N.: SIGIR 2017 workshop on open knowledge base and question answering (OKBQA 2017). In: Proceedings of the ACM SIGIR, pp. 1433\u20131434 (2017)","DOI":"10.1145\/3077136.3084372"},{"issue":"2","key":"18_CR5","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1016\/j.ipm.2014.10.006","volume":"51","author":"L Derczynski","year":"2015","unstructured":"Derczynski, L., et al.: Analysis of named entity recognition and linking for tweets. Inf. Process. Manag. 51(2), 32\u201349 (2015)","journal-title":"Inf. Process. Manag."},{"key":"18_CR6","unstructured":"Diefenbach, D., Both, A., Singh, K., Maret, P.: Towards a question answering system over the semantic web. arXiv preprint arXiv:1803.00832 (2018)"},{"key":"18_CR7","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"108","DOI":"10.1007\/978-3-030-00671-6_7","volume-title":"The Semantic Web \u2013 ISWC 2018","author":"M Dubey","year":"2018","unstructured":"Dubey, M., Banerjee, D., Chaudhuri, D., Lehmann, J.: EARL: joint entity and relation linking for question answering over knowledge graphs. In: Vrande\u010di\u0107, D., et al. (eds.) ISWC 2018. LNCS, vol. 11136, pp. 108\u2013126. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-030-00671-6_7"},{"key":"18_CR8","doi-asserted-by":"crossref","unstructured":"Ferragina, P., Scaiella, U.: TAGME: on-the-fly annotation of short text fragments (by Wikipedia entities). In: Proceedings of the 19th ACM Conference on Information and Knowledge Management, CIKM 2010, Toronto, Ontario, Canada, 26\u201330 October 2010, pp. 1625\u20131628. ACM (2010)","DOI":"10.1145\/1871437.1871689"},{"issue":"6","key":"18_CR9","doi-asserted-by":"publisher","first-page":"895","DOI":"10.3233\/SW-160247","volume":"8","author":"K H\u00f6ffner","year":"2017","unstructured":"H\u00f6ffner, K., Walter, S., Marx, E., Usbeck, R., Lehmann, J., Ngomo, A.N.: Survey on challenges of question answering in the semantic web. Semant. Web 8(6), 895\u2013920 (2017)","journal-title":"Semant. Web"},{"key":"18_CR10","doi-asserted-by":"crossref","unstructured":"Huang, X., Zhang, J., Li, D., Li, P.: Knowledge graph embedding based question answering. In: Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining, pp. 105\u2013113. ACM (2019)","DOI":"10.1145\/3289600.3290956"},{"issue":"1","key":"18_CR11","first-page":"73","volume":"8","author":"F Li","year":"2014","unstructured":"Li, F., Jagadish, H.V.: Constructing an interactive natural language interface for relational databases. PVLDB 8(1), 73\u201384 (2014)","journal-title":"PVLDB"},{"key":"18_CR12","unstructured":"Loni, B.: A survey of state-of-the-art methods on question classification (2011)"},{"key":"18_CR13","doi-asserted-by":"crossref","unstructured":"Maheshwari, G., Trivedi, P., Lukovnikov, D., Chakraborty, N., Fischer, A., Lehmann, J.: Learning to rank query graphs for complex question answering over knowledge graphs. arXiv preprint arXiv:1811.01118 (2018)","DOI":"10.1007\/978-3-030-30793-6_28"},{"key":"18_CR14","unstructured":"Mazzeo, G.M., Zaniolo, C.: Answering controlled natural language questions on RDF knowledge bases. In: EDBT, pp. 608\u2013611 (2016)"},{"key":"18_CR15","doi-asserted-by":"crossref","unstructured":"Mendes, P.N., Jakob, M., Garc\u00eda-Silva, A., Bizer, C.: DBpedia spotlight: shedding light on the web of documents. In: Proceedings the 7th International Conference on Semantic Systems, I-SEMANTICS 2011, Graz, Austria, 7\u20139 September 2011, pp. 1\u20138. ACM (2011)","DOI":"10.1145\/2063518.2063519"},{"issue":"2","key":"18_CR16","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1145\/763693.763694","volume":"21","author":"D Moldovan","year":"2003","unstructured":"Moldovan, D., Pa\u015fca, M., Harabagiu, S., Surdeanu, M.: Performance issues and error analysis in an open-domain question answering system. ACM Trans. Inf. Syst. (TOIS) 21(2), 133\u2013154 (2003)","journal-title":"ACM Trans. Inf. Syst. (TOIS)"},{"key":"18_CR17","unstructured":"Ngomo, N.: 9th challenge on question answering over linked data (QALD-9). Language 7:1"},{"key":"18_CR18","doi-asserted-by":"crossref","unstructured":"Sakor, A., et al.: Old is gold: linguistic driven approach for entity and relation linking of short text. In: NAACL 2019. ACL (2019, to appear)","DOI":"10.18653\/v1\/N19-1243"},{"key":"18_CR19","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"261","DOI":"10.1007\/978-3-319-25010-6_15","volume-title":"The Semantic Web - ISWC 2015","author":"M Saleem","year":"2015","unstructured":"Saleem, M., Ali, M.I., Hogan, A., Mehmood, Q., Ngomo, A.-C.N.: LSQ: the linked SPARQL queries dataset. In: Arenas, M., et al. (eds.) ISWC 2015. LNCS, vol. 9367, pp. 261\u2013269. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-25010-6_15"},{"key":"18_CR20","unstructured":"Saleem, M., Dastjerdi, S.N., Usbeck, R., Ngomo, A.N.: Question answering over linked data: what is difficult to answer? What affects the F scores? In: Joint Proceedings of BLINK 2017: Co-Located with (ISWC 2017), Austria (2017)"},{"key":"18_CR21","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1007\/978-3-319-25007-6_4","volume-title":"The Semantic Web - ISWC 2015","author":"M Saleem","year":"2015","unstructured":"Saleem, M., Mehmood, Q., Ngonga Ngomo, A.-C.: FEASIBLE: a feature-based SPARQL benchmark generation framework. In: Arenas, M., et al. (eds.) ISWC 2015. LNCS, vol. 9366, pp. 52\u201369. Springer, Cham (2015). https:\/\/doi.org\/10.1007\/978-3-319-25007-6_4"},{"key":"18_CR22","doi-asserted-by":"crossref","unstructured":"Saleem, M., Stadler, C., Mehmood, Q., Lehmann, J., Ngomo, A.-C.N.: SQCFramework: SPARQL query containment benchmark generation framework. In: Proceedings of the Knowledge Capture Conference, p. 28. ACM (2017)","DOI":"10.1145\/3148011.3148017"},{"key":"18_CR23","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1016\/j.websem.2014.06.002","volume":"30","author":"S Shekarpour","year":"2015","unstructured":"Shekarpour, S., Marx, E., Ngomo, A.N., Auer, S.: SINA: semantic interpretation of user queries for question answering on interlinked data. J. Web Sem. 30, 39\u201351 (2015)","journal-title":"J. Web Sem."},{"key":"18_CR24","unstructured":"Singh, K.: Towards dynamic composition of question answering pipelines. Ph.D. thesis, University of Bonn, Germany (2019)"},{"key":"18_CR25","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"624","DOI":"10.1007\/978-3-319-93417-4_40","volume-title":"The Semantic Web","author":"K Singh","year":"2018","unstructured":"Singh, K., Both, A., Sethupat, A., Shekarpour, S.: Frankenstein: a platform enabling reuse of question answering components. In: Gangemi, A., et al. (eds.) ESWC 2018. LNCS, vol. 10843, pp. 624\u2013638. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-93417-4_40"},{"key":"18_CR26","unstructured":"Singh, K., Lytra, I., Radhakrishna, A.S., Shekarpour, S., Vidal, M.-E., Lehmann, J.: No one is perfect: Analysing the performance of question answering components over the DBpedia knowledge graph. arXiv:1809.10044 (2018)"},{"key":"18_CR27","doi-asserted-by":"crossref","unstructured":"Singh, K., et al.: Why reinvent the wheel: let\u2019s build question answering systems together. In: Web Conference, pp. 1247\u20131256 (2018)","DOI":"10.1145\/3178876.3186023"},{"key":"18_CR28","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"210","DOI":"10.1007\/978-3-319-68204-4_22","volume-title":"The Semantic Web \u2013 ISWC 2017","author":"P Trivedi","year":"2017","unstructured":"Trivedi, P., Maheshwari, G., Dubey, M., Lehmann, J.: LC-QuAD: a corpus for complex question answering over knowledge graphs. In: d\u2019Amato, C., et al. (eds.) ISWC 2017. LNCS, vol. 10588, pp. 210\u2013218. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-68204-4_22"},{"key":"18_CR29","unstructured":"Unger, C., et al.: Question answering over linked data (QALD-5). In: Working Notes of CLEF 2015 - Conference and Labs of the Evaluation forum, Toulouse, France, 8\u201311 September 2015. CEUR-WS.org (2015)"},{"key":"18_CR30","unstructured":"Usbeck, R., Hoffmann, M., R\u00f6der, M., Lehmann, J., Ngomo, A.N.: Using multi-label classification for improved question answering. CoRR (2017)"},{"key":"18_CR31","doi-asserted-by":"crossref","unstructured":"Usbeck, R., et al.: Benchmarking question answering systems. Semant. Web J. (2019)","DOI":"10.3233\/SW-180312"},{"key":"18_CR32","doi-asserted-by":"crossref","unstructured":"Usbeck, R., et al.: GERBIL: general entity annotator benchmarking framework. In: WWW 2015, pp. 1133\u20131143 (2015)","DOI":"10.1145\/2736277.2741626"},{"key":"18_CR33","unstructured":"Voorhees, E.M., Harman, D.K. (eds.): Proceedings of The Eighth Text REtrieval Conference, TREC 1999, Gaithersburg, Maryland, USA, 17\u201319 November 1999, volume Special Publication, 500-246. National Institute of Standards and Technology (NIST) (1999)"},{"key":"18_CR34","doi-asserted-by":"crossref","unstructured":"Vrandecic, D.: Wikidata: a new platform for collaborative data collection. In: Proceedings of the 21st World Wide Web Conference, WWW 2012, Lyon, France, 16\u201320 April 2012 (Companion Volume), pp. 1063\u20131064. ACM (2012)","DOI":"10.1145\/2187980.2188242"},{"issue":"2","key":"18_CR35","doi-asserted-by":"publisher","first-page":"385","DOI":"10.3233\/SW-180334","volume":"10","author":"J Waitelonis","year":"2019","unstructured":"Waitelonis, J., J\u00fcrges, H., Sack, H.: Remixing entity linking evaluation datasets for focused benchmarking. Semant. Web 10(2), 385\u2013412 (2019)","journal-title":"Semant. Web"},{"key":"18_CR36","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"714","DOI":"10.1007\/978-3-319-93417-4_46","volume-title":"The Semantic Web","author":"H Zafar","year":"2018","unstructured":"Zafar, H., Napolitano, G., Lehmann, J.: Formal query generation for question answering over knowledge bases. In: Gangemi, A., et al. (eds.) ESWC 2018. LNCS, vol. 10843, pp. 714\u2013728. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-319-93417-4_46"}],"container-title":["Lecture Notes in Computer Science","The Semantic Web \u2013 ISWC 2019"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-30796-7_18","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,16]],"date-time":"2024-10-16T23:06:21Z","timestamp":1729119981000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-30796-7_18"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030307950","9783030307967"],"references-count":36,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-30796-7_18","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"17 October 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ISWC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Semantic Web Conference","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Auckland","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"New Zealand","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"26 October 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 October 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"semweb2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/iswc2019.semanticweb.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single- and Double blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"283","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"74","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"26% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3-4","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1-3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}