{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T03:43:12Z","timestamp":1779162192942,"version":"3.51.4"},"reference-count":61,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2015,4,30]],"date-time":"2015-04-30T00:00:00Z","timestamp":1430352000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/2.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["BMC Bioinformatics"],"published-print":{"date-parts":[[2015,12]]},"DOI":"10.1186\/s12859-015-0564-6","type":"journal-article","created":{"date-parts":[[2015,4,29]],"date-time":"2015-04-29T03:05:00Z","timestamp":1430276700000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":334,"title":["An overview of the BIOASQ large-scale biomedical semantic indexing and question answering competition"],"prefix":"10.1186","volume":"16","author":[{"given":"George","family":"Tsatsaronis","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Georgios","family":"Balikas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Prodromos","family":"Malakasiotis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ioannis","family":"Partalas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthias","family":"Zschunke","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael R","family":"Alvers","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dirk","family":"Weissenborn","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anastasia","family":"Krithara","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sergios","family":"Petridis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dimitris","family":"Polychronopoulos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yannis","family":"Almirantis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"John","family":"Pavlopoulos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nicolas","family":"Baskiotis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Patrick","family":"Gallinari","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thierry","family":"Arti\u00e9res","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Axel-Cyrille Ngonga","family":"Ngomo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Norman","family":"Heino","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Eric","family":"Gaussier","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liliana","family":"Barrio-Alvers","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Schroeder","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ion","family":"Androutsopoulos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Georgios","family":"Paliouras","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2015,4,30]]},"reference":[{"key":"564_CR1","unstructured":"The BioASQ Challenge. http:\/\/www.bioasq.org\/ ."},{"key":"564_CR2","doi-asserted-by":"publisher","first-page":"783","DOI":"10.1093\/nar\/gki470","volume":"33","author":"A Doms","year":"2005","unstructured":"Doms A, Schroeder M. GoPubMed: exploring PubMed with the Gene Ontology. Nucleic Acids Res. 2005; 33:783\u20136.","journal-title":"Nucleic Acids Res."},{"key":"564_CR3","doi-asserted-by":"publisher","first-page":"31","DOI":"10.1007\/s10618-010-0175-9","volume":"22","author":"AA Freitas","year":"2011","unstructured":"Silla Jr CN, Freitas AA. A survey of hierarchical classification across different application domains. Data Mining Knowledge Discovery. 2011; 22:31\u201372.","journal-title":"Data Mining Knowledge Discovery"},{"key":"564_CR4","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.cmpb.2009.10.003","volume":"99","author":"SJ Athenikos","year":"2010","unstructured":"Athenikos SJ, Han H. Biomedical question answering: A survey. Computer Methods and Programs in Biomedicine. 2010; 99:1\u201324.","journal-title":"Computer Methods and Programs in Biomedicine"},{"issue":"1","key":"564_CR5","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1504\/IJMSO.2007.015073","volume":"2","author":"C Mangold","year":"2007","unstructured":"Mangold C. A survey and classification of semantic search approaches. IJMSO. 2007; 2(1):23\u201334.","journal-title":"IJMSO"},{"key":"564_CR6","doi-asserted-by":"crossref","unstructured":"Abacha AB, Zweigenbaum P. Medical question answering: translating medical questions into sparql queries. In: ACM International Health Informatics Symposium, IHI \u201912, Miami, FL, USA, January 28\u201330, 2012: 2012. p. 41\u201350. http:\/\/doi.acm.org\/10.1145\/2110363.2110372 .","DOI":"10.1145\/2110363.2110372"},{"key":"564_CR7","unstructured":"TREC Question Answering Track. http:\/\/trec.nist.gov\/data\/qamain.html ."},{"key":"564_CR8","doi-asserted-by":"publisher","first-page":"361","DOI":"10.1017\/S1351324901002789","volume":"7","author":"EM Voorhees","year":"2001","unstructured":"Voorhees EM. The TREC question answering track. Natural Language Eng. 2001; 7:361\u2013378.","journal-title":"Natural Language Eng."},{"key":"564_CR9","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10791-008-9076-6","volume":"12","author":"W Hersh","year":"2009","unstructured":"Hersh W, Voorhees E. TREC genomics special issue overview. Inf Retrieval. 2009; 12:1\u201315.","journal-title":"Inf Retrieval."},{"key":"564_CR10","unstructured":"Text Analysis Conference. http:\/\/www.nist.gov\/tac\/ ."},{"key":"564_CR11","unstructured":"The NTCIR project. http:\/\/research.nii.ac.jp\/ntcir\/index-en.html ."},{"key":"564_CR12","unstructured":"Forner P, Giampiccolo D, Magnini B, Penas A, Rodrigo A, Sutcliffe RFE. Evaluating multilingual question answering systems at CLEF. In: Proceedings of the International Conference on Language Resources and Evaluation: 2010. Paris, France: European Language Resources Association."},{"key":"564_CR13","unstructured":"The Cross Language Evaluation Forum. http:\/\/www.clef-campaign.org\/ ."},{"key":"564_CR14","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1145\/1842890.1842894","volume":"44","author":"A Kosmopoulos","year":"2010","unstructured":"Kosmopoulos A, Gaussier E, Paliouras G, Aseervaatham S. The ECIR 2010 large scale hierarchical classification workshop. SIGIR Forum. 2010; 44:23\u201332.","journal-title":"SIGIR Forum."},{"key":"564_CR15","unstructured":"The Large Scale Hierarchical Text Classification Challenge. http:\/\/lshtc.iit.demokritos.gr\/ ."},{"key":"564_CR16","unstructured":"The JRS 2012 Data Mining Competition. http:\/\/tunedit.org\/challenge\/JRS12Contest ."},{"key":"564_CR17","unstructured":"The BioNLP Shared Task 2011. https:\/\/sites.google.com\/site\/bionlpst\/ ."},{"key":"564_CR18","unstructured":"BioCreative. http:\/\/www.biocreative.org\/ ."},{"key":"564_CR19","unstructured":"Huang M, Lu Z. Learning to annotate scientific publications. In: COLING 2010, 23rd International Conference on Computational Linguistics, Posters Volume, 23\u201327 August 2010, Beijing, China: 2010. p. 463\u201371. http:\/\/aclweb.org\/anthology-new\/C\/C10\/C10--2053.pdf ."},{"issue":"5","key":"564_CR20","first-page":"660","volume":"18","author":"M Huang","year":"2011","unstructured":"Huang M, N\u00e9v\u00e9ol A, Lu Z. Recommending mesh terms for annotating biomedical articles. JAMIA. 2011; 18(5):660\u20137.","journal-title":"JAMIA"},{"key":"564_CR21","unstructured":"Wahle M, Widdows D, Herskovic JR, Bernstam EV, Cohen T. Deterministic binary vectors for efficient automated indexing of medline\/pubmed abstracts. In: AMIA 2012, American Medical Informatics Association Annual Symposium, Chicago, Illinois, USA, November 3-7, 2012: 2012. http:\/\/knowledge.amia.org\/amia-55142-a2012a-1.636547\/t-003-1.640625\/f-001-1.640626\/a-107-1.640909\/a-108-1.640906 ."},{"key":"564_CR22","doi-asserted-by":"publisher","first-page":"208","DOI":"10.1186\/1471-2105-14-208","volume":"14","author":"AJ Jimeno-Yepes","year":"2013","unstructured":"Jimeno-Yepes AJ, Plaza L, Mork JG, Aronson AR, D\u00edaz A. Mesh indexing based on automatically generated summaries. BMC Bioinf. 2013; 14:208.","journal-title":"BMC Bioinf"},{"key":"564_CR23","unstructured":"The JavaScript Object Notation. http:\/\/www.json.com\/ ."},{"key":"564_CR24","doi-asserted-by":"crossref","unstructured":"Tsoumakas G, Katakis I, Vlahavas IP. Mining multi-label data. In: Data Mining and Knowledge Discovery Handbook: 2010. p. 667\u201385.","DOI":"10.1007\/978-0-387-09823-4_34"},{"key":"564_CR25","first-page":"1","volume":"29","author":"A Kosmopoulos","year":"2014","unstructured":"Kosmopoulos A, Partalas I, Gaussier E, Paliouras G, Androutsopoulos I. Evaluation measures for hierarchical classification: a unified view and novel approaches. Data Mining and Knowledge Discovery. 2014; 29:1\u201346.","journal-title":"Data Mining and Knowledge Discovery"},{"key":"564_CR26","unstructured":"Doms A. GoPubMed: Ontology-based literature search for the life sciences: PhD thesis, Technical University of Dresden; 2008."},{"issue":"4","key":"564_CR27","doi-asserted-by":"publisher","first-page":"482","DOI":"10.1016\/0196-8858(81)90046-4","volume":"2","author":"TF Smith","year":"1981","unstructured":"Smith TF, Waterman MS. Comparison of biosequences. Adv Appl Mathematics. 1981; 2(4):482\u20139.","journal-title":"Adv Appl Mathematics"},{"key":"564_CR28","volume-title":"BioASQ@CLEF","author":"JG Mork","year":"2013","unstructured":"Mork JG, Jimeno-Yepes A, Aronson AR. The nlm medical text indexer system for indexing biomedical literature. In: BioASQ@CLEF. CEUR Workshop Proceedings: Aachen, Germany: 2013."},{"key":"564_CR29","unstructured":"Lin C-Y. ROUGE: A package for automatic evaluation of summaries. In: Proceedings of the ACL Workshop \u2018Text Summarization Branches Out\u2019. PA, USA: 2004. p. 74\u201381."},{"key":"564_CR30","volume-title":"COLING","author":"D Galanis","year":"2012","unstructured":"Galanis D, Lampouras G, Androutsopoulos I. Extractive multi-document summarization with integer linear programming and support vector regression. In: COLING. India: Indian Institute of Technology Bombay: 2012. p. 911\u201326."},{"key":"564_CR31","volume-title":"BioASQ@CLEF","author":"D Weissenborn","year":"2013","unstructured":"Weissenborn D, Tsatsaronis G, Schroeder M. Answering factoid questions in the biomedical domain. In: BioASQ@CLEF. Aachen, Germany: CEUR Workshop Proceedings: 2013."},{"key":"564_CR32","volume-title":"1st BioASQ Workshop: A Challenge on Large-scale Biomedical Semantic Indexing and Question Answering","author":"G Tsoumakas","year":"2013","unstructured":"Tsoumakas G, Laliotis M, Markontanatos N, Vlahavas I. Large-scale semantic indexing of biomedical publications. In: 1st BioASQ Workshop: A Challenge on Large-scale Biomedical Semantic Indexing and Question Answering. Aachen, Germany: CEUR Workshop Proceedings: 2013."},{"key":"564_CR33","unstructured":"Mork JG, Demner-Fushman D, Schmidt S, Aronson AR. Recent enhancements to the NLM medical text indexer. In: Working Notes for CLEF 2014 Conference, Sheffield, UK, September 15\u201318, 2014: 2014. p. 1328\u20131336. http:\/\/ceur-ws.org\/Vol-1180\/CLEF2014wn-QA-MorkEt2014.pdf ."},{"key":"564_CR34","first-page":"1","volume":"7","author":"J Demsar","year":"2006","unstructured":"Demsar J. Statistical comparisons of classifiers over multiple data sets. J Machine Learning Res. 2006; 7:1\u201330.","journal-title":"J Machine Learning Res"},{"key":"564_CR35","unstructured":"Mork J, Jimeno-Yepes A, Aronson A. September 27th, 2013. The NLM Medical text indexer system for indexing biomedical literature. Valencia, Spain. http:\/\/ceur-ws.org\/Vol-1094\/bioasq2013_submission_3.pdf."},{"key":"564_CR36","unstructured":"BioASQ Task 1a. http:\/\/bioasq.lip6.fr\/general_information\/Task1a\/ ."},{"key":"564_CR37","volume-title":"1st BioASQ Workshop: A Challenge on Large-scale Biomedical Semantic Indexing and Question Answering","author":"F Ribadas","year":"2013","unstructured":"Ribadas F, de Campos L, Darriba V, Romero A. Two hierarchical text categorization approaches for bioasq semantic indexing challenge. In: 1st BioASQ Workshop: A Challenge on Large-scale Biomedical Semantic Indexing and Question Answering. Aachen, Germany: CEUR Workshop Proceedings: 2013."},{"key":"564_CR38","unstructured":"Balikas G, Partalas I, Kosmopoulos A, Petridis S, Malakasiotis P, Pavlopoulos I, et al.Bioasq evaluation framework specifications. Project deliverable D4.1 (May 2013). http:\/\/bioasq.org\/sites\/default\/files\/PublicDocuments\/BioASQ_D4.1-EvaluationFrameworkSpecification_final.pdf ."},{"key":"564_CR39","unstructured":"Malakasiotis P, Androutsopoulos I, Pavlopoulos I. Challenge evaluation report. Project deliverable D5.2 (December 2013). http:\/\/www.bioasq.org\/sites\/default\/files\/PublicDocuments\/BioASQ_D5.2-ChallengeEvaluation_final.pdf ."},{"key":"564_CR40","unstructured":"The PONTE EU Project. http:\/\/www.ponte-project.eu\/ ."},{"key":"564_CR41","unstructured":"The BioASQ Annotation Tool. http:\/\/at.bioasq.org ."},{"key":"564_CR42","unstructured":"The MongoDB. http:\/\/www.mongodb.org\/ ."},{"key":"564_CR43","unstructured":"Malakasiotis P, Androutsopoulos I, Almirantis Y, Polychronopoulos D, Pavlopoulos I. Tutorials and guidelines. Project deliverable D3.4 (January 2013). http:\/\/bioasq.org\/sites\/default\/files\/PublicDocuments\/2013-D3.4-TutorialsGuidelines.pdf ."},{"key":"564_CR44","volume-title":"Proceedings of the 18th International Conference on World Wide Web, WWW \u201909","author":"L Tang","year":"2009","unstructured":"Tang L, Rajan S, Narayanan VK. Large scale multi-label classification via metalabeler. In: Proceedings of the 18th International Conference on World Wide Web, WWW \u201909. New York, NY, USA: ACM: 2009. p. 211\u2013220."},{"key":"564_CR45","unstructured":"Zhu D, Li D, Carterette B, Liu H. An incemental approach for medline mesh indexing. In: 1st BioASQ Workshop: A Challenge on Large-scale Biomedical Semantic Indexing and Question Answering. Aachen, Germany:CEUR Workshop Proceedings; 2013."},{"key":"564_CR46","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1136\/jamia.2009.002733","volume":"17","author":"AR Aronson","year":"2010","unstructured":"Aronson AR, Lang F-M. An overview of metamap: historical perspective and recent advances. J Am Med Inf Assoc. 2010; 17:229\u201336.","journal-title":"J Am Med Inf Assoc"},{"key":"564_CR47","unstructured":"The Indri Engine. http:\/\/www.lemurproject.org\/indri.php ."},{"key":"564_CR48","unstructured":"Liu Y. Bioasq system descriptions (wishart team). Technical report. 2013."},{"key":"564_CR49","unstructured":"Mao Y, Lu Z. Ncbi at the 2013 bioasq challenge task: Learning to rank for automatic mesh indexing. Technical report. 2013."},{"key":"564_CR50","unstructured":"The PolySearch Tool. http:\/\/wishart.biology.ualberta.ca\/polysearch\/ ."},{"key":"564_CR51","volume-title":"CIKM","author":"S Robertson","year":"2006","unstructured":"Robertson S. On gmap: and other transformations. In: CIKM. NY, USA: ACM: 2006. p. 78\u201383."},{"issue":"4","key":"564_CR52","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1561\/1500000009","volume":"4","author":"M Sanderson","year":"2010","unstructured":"Sanderson M. Test collection based evaluation of information retrieval systems. Found Trends Inf Retrieval. 2010; 4(4):247\u2013375.","journal-title":"Found Trends Inf Retrieval"},{"key":"564_CR53","unstructured":"The Joint Chemical Dictionary. http:\/\/biosemantics.org\/index.php\/resources\/jochem ."},{"issue":"22","key":"564_CR54","doi-asserted-by":"publisher","first-page":"2983","DOI":"10.1093\/bioinformatics\/btp535","volume":"25","author":"KM Hettne","year":"2009","unstructured":"Hettne KM, Stierum RH, Schuemie MJ, Hendriksen PJM, Schijvenaars BJA, van Mulligen EM, et al.A dictionary to identify small molecules and drugs in free text. Bioinformatics. 2009; 25(22):2983\u201391.","journal-title":"Bioinformatics"},{"key":"564_CR55","unstructured":"The Gene Ontology. http:\/\/www.geneontology.org\/ ."},{"key":"564_CR56","unstructured":"UniProt. http:\/\/www.uniprot.org\/ ."},{"key":"564_CR57","unstructured":"The Disease Ontology Wiki. http:\/\/do-wiki.nubic.northwestern.edu\/do-wiki\/index.php\/Main_Page ."},{"key":"564_CR58","unstructured":"MeSH. http:\/\/www.nlm.nih.gov\/mesh\/ ."},{"key":"564_CR59","unstructured":"PubMed. http:\/\/www.ncbi.nlm.nih.gov\/pubmed\/ ."},{"key":"564_CR60","unstructured":"PMC. http:\/\/www.ncbi.nlm.nih.gov\/pmc\/ ."},{"key":"564_CR61","unstructured":"The Linked Life Data Project. http:\/\/linkedlifedata.com\/ ."}],"container-title":["BMC Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1186\/s12859-015-0564-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s12859-015-0564-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s12859-015-0564-6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/s12859-015-0564-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,27]],"date-time":"2025-05-27T17:07:38Z","timestamp":1748365658000},"score":1,"resource":{"primary":{"URL":"https:\/\/bmcbioinformatics.biomedcentral.com\/articles\/10.1186\/s12859-015-0564-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,4,30]]},"references-count":61,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2015,12]]}},"alternative-id":["564"],"URL":"https:\/\/doi.org\/10.1186\/s12859-015-0564-6","relation":{},"ISSN":["1471-2105"],"issn-type":[{"value":"1471-2105","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,4,30]]},"assertion":[{"value":"19 February 2014","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"31 March 2015","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"30 April 2015","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"138"}}