{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,23]],"date-time":"2026-03-23T23:43:53Z","timestamp":1774309433057,"version":"3.50.1"},"publisher-location":"Cham","reference-count":48,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030852504","type":"print"},{"value":"9783030852511","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-85251-1_18","type":"book-chapter","created":{"date-parts":[[2021,9,14]],"date-time":"2021-09-14T00:11:11Z","timestamp":1631578271000},"page":"239-263","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":18,"title":["Overview of BioASQ 2021: The Ninth BioASQ Challenge on Large-Scale Biomedical Semantic Indexing and\u00a0Question Answering"],"prefix":"10.1007","author":[{"given":"Anastasios","family":"Nentidis","sequence":"first","affiliation":[]},{"given":"Georgios","family":"Katsimpras","sequence":"additional","affiliation":[]},{"given":"Eirini","family":"Vandorou","sequence":"additional","affiliation":[]},{"given":"Anastasia","family":"Krithara","sequence":"additional","affiliation":[]},{"given":"Luis","family":"Gasco","sequence":"additional","affiliation":[]},{"given":"Martin","family":"Krallinger","sequence":"additional","affiliation":[]},{"given":"Georgios","family":"Paliouras","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,9,14]]},"reference":[{"key":"18_CR1","unstructured":"attentionxml: Label tree-based attention-aware deep model for high-performance extreme multi-label text classification"},{"key":"18_CR2","unstructured":"ku-dmis at bioasq 9: Data-centric and model-centric approaches for biomedical question answering"},{"key":"18_CR3","unstructured":"Almeida, T., Matos, S.: BIT.UA at BioASQ 8: lightweight neural document ranking with zero-shot snippet retrieval. In: CLEF (Working Notes) (2020)"},{"key":"18_CR4","unstructured":"Almeida, T., Matos, S.: BioASQ synergy: a strong and simple baseline rooted in relevance feedback. In: CLEF (Working Notes) (2021)"},{"key":"18_CR5","unstructured":"Almeida, T., Matos, S.: Universal passage weighting mechanism (UPWM) in BioASQ 9b. In: CLEF (Working Notes) (2021)"},{"key":"18_CR6","unstructured":"Alrowili, S., Shanker, K.: Large biomedical question answering models with ALBERT and ELECTRA. In: CLEF (Working Notes) (2021)"},{"key":"18_CR7","doi-asserted-by":"crossref","unstructured":"Alrowili, S., Shanker, V.: BioM-transformers: building large biomedical language models with BERT, ALBERT and ELECTRA. In: Proceedings of the 20th Workshop on Biomedical Language Processing, pp. 221\u2013227. Association for Computational Linguistics, June 2021. https:\/\/www.aclweb.org\/anthology\/2021.bionlp-1.24","DOI":"10.18653\/v1\/2021.bionlp-1.24"},{"key":"18_CR8","unstructured":"Baldwin, B., Carpenter, B.: LingPipe. World Wide Web (2003). http:\/\/alias-i.com\/lingpipe"},{"key":"18_CR9","unstructured":"Balikas, G., et al.: Evaluation framework specifications. Project deliverable D4.1, UPMC, May 2013"},{"key":"18_CR10","unstructured":"Campos, M., Couto, F.: Post-processing BioBERT and using voting methods for biomedical question answering. In: CLEF (Working Notes) (2021)"},{"key":"18_CR11","unstructured":"Clark, K., Luong, M.T., Le, Q.V., Manning, C.D.: ELECTRA: pre-training text encoders as discriminators rather than generators. arXiv preprint arXiv:2003.10555 (2020)"},{"key":"18_CR12","first-page":"1","volume":"7","author":"J Demsar","year":"2006","unstructured":"Demsar, J.: Statistical comparisons of classifiers over multiple data sets. J. Mach. Learn. Res. 7, 1\u201330 (2006)","journal-title":"J. Mach. Learn. Res."},{"key":"18_CR13","unstructured":"Garc\u00eda-Pablos, A., Perez, N., Cuadros, M.: Vicomtech at MESINESP2: BERT-based multi-label classification models for biomedical text indexing (2021)"},{"key":"18_CR14","unstructured":"Gasco, L., et al.: Overview of BioASQ 2021-MESINESP track. Evaluation of advance hierarchical classification techniques for scientific literature, patents and clinical trials (2021)"},{"key":"18_CR15","unstructured":"Huang, Y., Buse, G., Abdullatif, K., Ozgur, A., Ozkirimli, E.: Pidna at BioASQ MESINESP: hybrid semantic indexing for biomedical articles in Spanish (2021)"},{"key":"18_CR16","unstructured":"Khanna, U., Molla, D.: Transformer-based language models for factoid question answering at bioasq9b. In: CLEF (Working Notes) (2021)"},{"issue":"3","key":"18_CR17","doi-asserted-by":"publisher","first-page":"820","DOI":"10.1007\/s10618-014-0382-x","volume":"29","author":"A Kosmopoulos","year":"2015","unstructured":"Kosmopoulos, A., Partalas, I., Gaussier, E., Paliouras, G., Androutsopoulos, I.: Evaluation measures for hierarchical classification: a unified view and novel approaches. Data Min. Knowl. Disc. 29(3), 820\u2013865 (2015)","journal-title":"Data Min. Knowl. Disc."},{"key":"18_CR18","unstructured":"Krallinger, M., et al.: Overview of the CHEMDNER patents task. In: Proceedings of the Fifth BioCreative Challenge Evaluation Workshop, pp. 63\u201375 (2015)"},{"key":"18_CR19","unstructured":"Miranda-Escalada, A., Farr\u00e9, E., Krallinger, M.: Named entity recognition, concept normalization and clinical coding: Overview of the cantemist track for cancer text mining in Spanish, corpus, guidelines, methods and results. In: Proceedings of the Iberian Languages Evaluation Forum (IberLEF 2020). CEUR Workshop Proceedings (2020)"},{"key":"18_CR20","doi-asserted-by":"crossref","unstructured":"Miranda-Escalada, A.: The ProfNER shared task on automatic recognition of occupation mentions in social media: systems, evaluation, guidelines, embeddings and corpora. In: Proceedings of the Sixth Social Media Mining for Health (# SMM4H) Workshop and Shared Task, pp. 13\u201320 (2021)","DOI":"10.18653\/v1\/2021.smm4h-1.3"},{"key":"18_CR21","unstructured":"Miranda-Escalada, A., Gonzalez-Agirre, A., Armengol-Estap\u00e9, J., Krallinger, M.: Overview of automatic clinical coding: annotations, guidelines, and solutions for non-English clinical cases at CodiEsp track of CLEF eHealth 2020. In: Working Notes of Conference and Labs of the Evaluation (CLEF) Forum. CEUR Workshop Proceedings (2020)"},{"key":"18_CR22","unstructured":"Molla, D., Jones, C., Nguyen, V.: Query focused multi-document summarisation of biomedical texts. arXiv preprint arXiv:2008.11986 (2020)"},{"key":"18_CR23","unstructured":"Molla, D., Khanna, U., Galat, D., Nguyen, V., Rybinski, M.: Query-focused extractive summarisation for finding ideal answers to biomedical and COVID-19 questions. In: CLEF (Working Notes) (2021)"},{"key":"18_CR24","unstructured":"Mork, J.G., Demner-Fushman, D., Schmidt, S.C., Aronson, A.R.: Recent enhancements to the NLM medical text indexer. In: Proceedings of Question Answering Lab at CLEF (2014)"},{"key":"18_CR25","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"194","DOI":"10.1007\/978-3-030-58219-7_16","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"A Nentidis","year":"2020","unstructured":"Nentidis, A., et al.: Overview of BioASQ 2020: the eighth BioASQ challenge on large-scale biomedical semantic indexing and question answering. In: Arampatzis, A., et al. (eds.) CLEF 2020. LNCS, vol. 12260, pp. 194\u2013214. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-58219-7_16"},{"key":"18_CR26","doi-asserted-by":"crossref","unstructured":"Ozyurt, I.B.: On the effectiveness of small, discriminatively pre-trained language representation models for biomedical text mining. In: Proceedings of the First Workshop on Scholarly Document Processing, pp. 104\u2013112 (2020)","DOI":"10.18653\/v1\/2020.sdp-1.12"},{"key":"18_CR27","unstructured":"Ozyurt, I.B.: End-to-end biomedical question answering via bio-answerfinder and discriminative language representation models. In: CLEF (Working Notes) (2021)"},{"key":"18_CR28","doi-asserted-by":"crossref","unstructured":"Ozyurt, I.B., Bandrowski, A., Grethe, J.S.: Bio-AnswerFinder: a system to find answers to questions from biomedical texts. Database 2020 (2020)","DOI":"10.1093\/database\/baz137"},{"key":"18_CR29","doi-asserted-by":"crossref","unstructured":"Pappas, D., Stavropoulos, P., Androutsopoulos, I.: AUEB-NLP at BioASQ 8: biomedical document and snippet retrieval (2020)","DOI":"10.1007\/978-3-030-43887-6_55"},{"issue":"12","key":"18_CR30","doi-asserted-by":"publisher","first-page":"i70","DOI":"10.1093\/bioinformatics\/btw294","volume":"32","author":"S Peng","year":"2016","unstructured":"Peng, S., You, R., Wang, H., Zhai, C., Mamitsuka, H., Zhu, S.: DeepMesh: deep semantic representation for improving large-scale mesh indexing. Bioinformatics 32(12), i70\u2013i79 (2016)","journal-title":"Bioinformatics"},{"key":"18_CR31","unstructured":"Rae, A., Mork, J., Demner-Fushman, D.: A neural text ranking approach for automatic mesh indexing. In: CLEF (Working Notes) (2021)"},{"key":"18_CR32","unstructured":"Rae, A.R., Pritchard, D.O., Mork, J.G., Demner-Fushman, D.: Automatic mesh indexing: revisiting the subheading attachment problem. In: AMIA Annual Symposium Proceedings, vol. 2020, p. 1031. American Medical Informatics Association (2020)"},{"key":"18_CR33","unstructured":"Raffel, C.: Exploring the limits of transfer learning with a unified text-to-text transformer. arXiv preprint arXiv:1910.10683 (2019)"},{"key":"18_CR34","unstructured":"Ribadas, F.J., De Campos, L.M., Darriba, V.M., Romero, A.E.: CoLe and UTAI at BioASQ 2015: experiments with similarity based descriptor assignment. CEUR Workshop Proc. 1391 (2015)"},{"key":"18_CR35","unstructured":"Rodriguez-Penagos, C.: Overview of MESINESP8, a Spanish medical semantic indexing task within BioASQ 2020 (2020)"},{"key":"18_CR36","unstructured":"Ruas, P., Andrade, V.D.T., Couto, F.M.: LASIGE-BioTM at MESINESP2: entity linking with semantic similarity and extreme multi-label classification on Spanish biomedical documents (2021)"},{"key":"18_CR37","unstructured":"Sarrouti, M., Gupta, D., Abacha, A.B., Demner-Fushman, D.: NLM at BioASQ 2021: deep learning-based methods for biomedical question answering about COVID-19. In: CLEF (Working Notes) (2021)"},{"key":"18_CR38","unstructured":"Torres-Salinas, D., Robinson-Garcia, N., van Schalkwyk, F., Nane, G.F., Castillo-Valdivieso, P.: The growth of COVID-19 scientific literature: a forecast analysis of different daily time series in specific settings. arXiv preprint arXiv:2101.12455 (2021)"},{"key":"18_CR39","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1186\/s12859-015-0564-6","volume":"16","author":"G Tsatsaronis","year":"2015","unstructured":"Tsatsaronis, G., et al.: An overview of the BioASQ large-scale biomedical semantic indexing and question answering competition. BMC Bioinform. 16, 138 (2015). https:\/\/doi.org\/10.1186\/s12859-015-0564-6","journal-title":"BMC Bioinform."},{"key":"18_CR40","unstructured":"Tsoumakas, G., Laliotis, M., Markontanatos, N., Vlahavas, I.: Large-scale semantic indexing of biomedical publications. In: 1st BioASQ Workshop: A Challenge on Large-Scale Biomedical Semantic Indexing and Question Answering (2013)"},{"key":"18_CR41","unstructured":"Wang, L.L., et al.: CORD-19: the COVID-19 open research dataset. ArXiv (2020)"},{"issue":"12","key":"18_CR42","doi-asserted-by":"publisher","first-page":"1907","DOI":"10.1093\/bioinformatics\/btv760","volume":"32","author":"CH Wei","year":"2016","unstructured":"Wei, C.H., Leaman, R., Lu, Z.: Beyond accuracy: creating interoperable and scalable text-mining web services. Bioinform. (Oxford, Engl.) 32(12), 1907\u201310 (2016). https:\/\/doi.org\/10.1093\/bioinformatics\/btv760","journal-title":"Bioinform. (Oxford, Engl.)"},{"key":"18_CR43","doi-asserted-by":"crossref","unstructured":"Xue, L., et al.: mT5: a massively multilingual pre-trained text-to-text transformer. arXiv preprint arXiv:2010.11934 (2020)","DOI":"10.18653\/v1\/2021.naacl-main.41"},{"key":"18_CR44","first-page":"23","volume":"2016","author":"Z Yang","year":"2016","unstructured":"Yang, Z., Zhou, Y., Eric, N.: Learning to answer biomedical questions: OAQA at BioASQ 4b. ACL 2016, 23 (2016)","journal-title":"ACL"},{"key":"18_CR45","doi-asserted-by":"crossref","unstructured":"Yoon, W., Jackson, R., Kang, J., Lagerberg, A.: Sequence tagging for biomedical extractive question answering. arXiv preprint arXiv:2104.07535 (2021)","DOI":"10.1093\/bioinformatics\/btac397"},{"issue":"5","key":"18_CR46","doi-asserted-by":"publisher","first-page":"684","DOI":"10.1093\/bioinformatics\/btaa837","volume":"37","author":"R You","year":"2021","unstructured":"You, R., Liu, Y., Mamitsuka, H., Zhu, S.: BERTMeSH: deep contextual representation learning for large-scale high-performance MeSH indexing with full text. Bioinformatics 37(5), 684\u2013692 (2021)","journal-title":"Bioinformatics"},{"key":"18_CR47","first-page":"8","volume":"2016","author":"I Zavorin","year":"2016","unstructured":"Zavorin, I., Mork, J.G., Demner-Fushman, D.: Using learning-to-rank to enhance NLM medical text indexer results. ACL 2016, 8 (2016)","journal-title":"ACL"},{"key":"18_CR48","unstructured":"Zhang, Y., Han, J.C., Tsai, R.T.H.: NCU-IISR\/AS-GIS: results of various pre-trained biomedical language models and logistic regression model in BioASQ task 9b phase b. In: CLEF (Working Notes) (2021)"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-85251-1_18","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,9]],"date-time":"2023-01-09T03:38:05Z","timestamp":1673235485000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-85251-1_18"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030852504","9783030852511"],"references-count":48,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-85251-1_18","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"14 September 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CLEF","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference of the Cross-Language Evaluation Forum for European Languages","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21 September 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"24 September 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"clef2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/clef2021.clef-initiative.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"21","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"11","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"52% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5 best of labs papers and 12 lab overview papers are also included.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}