{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T13:05:15Z","timestamp":1765544715606},"publisher-location":"Cham","reference-count":30,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030285760"},{"type":"electronic","value":"9783030285777"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-28577-7_26","type":"book-chapter","created":{"date-parts":[[2019,9,1]],"date-time":"2019-09-01T19:02:43Z","timestamp":1567364563000},"page":"322-339","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["Overview of the CLEF eHealth Evaluation Lab 2019"],"prefix":"10.1007","author":[{"given":"Liadh","family":"Kelly","sequence":"first","affiliation":[]},{"given":"Hanna","family":"Suominen","sequence":"additional","affiliation":[]},{"given":"Lorraine","family":"Goeuriot","sequence":"additional","affiliation":[]},{"given":"Mariana","family":"Neves","sequence":"additional","affiliation":[]},{"given":"Evangelos","family":"Kanoulas","sequence":"additional","affiliation":[]},{"given":"Dan","family":"Li","sequence":"additional","affiliation":[]},{"given":"Leif","family":"Azzopardi","sequence":"additional","affiliation":[]},{"given":"Rene","family":"Spijker","sequence":"additional","affiliation":[]},{"given":"Guido","family":"Zuccon","sequence":"additional","affiliation":[]},{"given":"Harrisen","family":"Scells","sequence":"additional","affiliation":[]},{"given":"Jo\u00e3o","family":"Palotti","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,8,3]]},"reference":[{"key":"26_CR1","first-page":"147","volume-title":"Engaging Patients with Health Information Technology","author":"M Adnan","year":"2015","unstructured":"Adnan, M., Warren, J., Suominen, H.: Patient empowerment via technologies for patient-friendly personalized language. In: Grando, A.M., Rozenblum, R., Bates, D. (eds.) Engaging Patients with Health Information Technology, pp. 147\u2013158. De Gruyter, Berlin (2015)"},{"issue":"12","key":"26_CR2","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pbio.2003217","volume":"15","author":"B Bert","year":"2017","unstructured":"Bert, B., et al.: Rethinking 3R strategies: digging deeper into animaltestinfo promotes transparency in in vivo biomedical research. PLOS Biol. 15(12), 1\u201320 (2017). \n                      https:\/\/doi.org\/10.1371\/journal.pbio.2003217","journal-title":"PLOS Biol."},{"doi-asserted-by":"publisher","unstructured":"Cormack, G.V., Grossman, M.R.: Engineering quality and reliability in technology-assisted review. In: Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2016, pp. 75\u201384. ACM, New York (2016). \n                      https:\/\/doi.org\/10.1145\/2911451.2911510","key":"26_CR3","DOI":"10.1145\/2911451.2911510"},{"unstructured":"Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. CoRR abs\/1810.04805 (2018). \n                      http:\/\/arxiv.org\/abs\/1810.04805","key":"26_CR4"},{"unstructured":"Goeuriot, L., et al.: ShARe\/CLEF eHealth Evaluation Lab 2013, Task 3: Information retrieval to address patients\u2019 questions when reading clinical reports. CLEF 2013 Online Working Notes 8138 (2013)","key":"26_CR5"},{"key":"26_CR6","doi-asserted-by":"publisher","first-page":"507","DOI":"10.1007\/s10791-018-9331-4","volume":"21","author":"L Goeuriot","year":"2018","unstructured":"Goeuriot, L., et al.: An analysis of evaluation campaigns in ad-hoc medical information retrieval: CLEF eHealth 2013 and 2014. Inf. Retrieval J. 21, 507\u2013540 (2018)","journal-title":"Inf. Retrieval J."},{"unstructured":"Goeuriot, L., et al.: ShARe\/CLEF eHealth Evaluation Lab 2014, Task 3: user-centred health information retrieval. In: CLEF 2014 Evaluation Labs and Workshop, Sheffield, UK, Online Working Notes (2014)","key":"26_CR7"},{"key":"26_CR8","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"429","DOI":"10.1007\/978-3-319-24027-5_44","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"L Goeuriot","year":"2015","unstructured":"Goeuriot, L., et al.: Overview of the CLEF eHealth evaluation lab 2015. In: Mothe, J., et al. (eds.) CLEF 2015. LNCS, vol. 9283, pp. 429\u2013443. Springer, Cham (2015). \n                      https:\/\/doi.org\/10.1007\/978-3-319-24027-5_44"},{"key":"26_CR9","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1007\/978-3-319-65813-1_26","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"L Goeuriot","year":"2017","unstructured":"Goeuriot, L., et al.: CLEF 2017 eHealth evaluation lab overview. In: Jones, G.J.F., et al. (eds.) CLEF 2017. LNCS, vol. 10456, pp. 291\u2013303. Springer, Cham (2017). \n                      https:\/\/doi.org\/10.1007\/978-3-319-65813-1_26"},{"unstructured":"Jimmy, Z.G., Palotti, J.: Overview of the clef 2018 consumer health search task. In: Working Notes of Conference and Labs of the Evaluation (CLEF) Forum, CEUR Workshop Proceedings (2018)","key":"26_CR10"},{"unstructured":"Kanoulas, E., Li, D., Azzopardi, L., Spijker, R.: CLEF 2017 technologically assisted reviews in empirical medicine overview. In: Working Notes of Conference and Labs of the Evaluation (CLEF) Forum, CEUR Workshop Proceedings (2017)","key":"26_CR11"},{"unstructured":"Kanoulas, E., Li, D., Azzopardi, L., Spijker, R.: CLEF 2018 technologically assisted reviews in empirical medicine overview. In: Working Notes of Conference and Labs of the Evaluation (CLEF) Forum, CEUR Workshop Proceedings (2018)","key":"26_CR12"},{"unstructured":"Kanoulas, E., Li, D., Azzopardi, L., Spijker, R.: CLEF 2019 technology assisted reviews in empirical medicine overview. In: Working Notes of Conference and Labs of the Evaluation (CLEF) Forum, CEUR Workshop Proceedings (2019)","key":"26_CR13"},{"key":"26_CR14","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"255","DOI":"10.1007\/978-3-319-44564-9_24","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"L Kelly","year":"2016","unstructured":"Kelly, L., Goeuriot, L., Suominen, H., N\u00e9v\u00e9ol, A., Palotti, J., Zuccon, G.: Overview of the CLEF eHealth evaluation lab 2016. In: Fuhr, N., et al. (eds.) CLEF 2016. LNCS, vol. 9822, pp. 255\u2013266. Springer, Cham (2016). \n                      https:\/\/doi.org\/10.1007\/978-3-319-44564-9_24"},{"key":"26_CR15","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"267","DOI":"10.1007\/978-3-030-15719-7_36","volume-title":"Advances in Information Retrieval","author":"L Kelly","year":"2019","unstructured":"Kelly, L., et al.: CLEF ehealth 2019 evaluation lab. In: Azzopardi, L., Stein, B., Fuhr, N., Mayr, P., Hauff, C., Hiemstra, D. (eds.) ECIR 2019. LNCS, vol. 11438, pp. 267\u2013274. Springer, Heidelberg (2019). \n                      https:\/\/doi.org\/10.1007\/978-3-030-15719-7_36"},{"key":"26_CR16","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"172","DOI":"10.1007\/978-3-319-11382-1_17","volume-title":"Information Access Evaluation. Multilinguality, Multimodality, and Interaction","author":"L Kelly","year":"2014","unstructured":"Kelly, L., et al.: Overview of the ShARe\/CLEF eHealth evaluation lab 2014. In: Kanoulas, E., et al. (eds.) CLEF 2014. LNCS, vol. 8685, pp. 172\u2013191. Springer, Cham (2014). \n                      https:\/\/doi.org\/10.1007\/978-3-319-11382-1_17"},{"unstructured":"Lee, J., et al.: BioBERT: a pre-trained biomedical language representation model for biomedical text mining. CoRR abs\/1901.08746 (2019). \n                      http:\/\/arxiv.org\/abs\/1901.08746","key":"26_CR17"},{"issue":"1","key":"26_CR18","doi-asserted-by":"publisher","first-page":"82","DOI":"10.1186\/2046-4053-2-82","volume":"2","author":"MM Leeflang","year":"2013","unstructured":"Leeflang, M.M., Deeks, J.J., Takwoingi, Y., Macaskill, P.: Cochrane diagnostic test accuracy reviews. Syst. Rev. 2(1), 82 (2013)","journal-title":"Syst. Rev."},{"unstructured":"N\u00e9v\u00e9ol, A., et al.: Clinical information extraction at the CLEF eHealth evaluation lab 2016. In: Balog, K., Cappellato, L., Ferro, N., Macdonald, C. (eds.) CLEF 2016 Working Notes, CEUR Workshop Proceedings. CEUR-WS.org, ISSN 1613\u20130073 (2016). \n                      http:\/\/ceur-ws.org\/Vol-1609\/","key":"26_CR19"},{"unstructured":"N\u00e9v\u00e9ol, A., et al.: CLEF eHealth 2017 multilingual information extraction task overview: ICD10 coding of death certificates in English and French. In: CLEF 2017 Online Working Notes. CEUR-WS (2017)","key":"26_CR20"},{"unstructured":"N\u00e9v\u00e9ol, A., et al.: CLEF eHealth 2018 multilingual information extraction task overview: ICD10 coding of death certificates in French, Hungarian and Italian. In: CLEF 2018 Online Working Notes. CEUR-WS (2018)","key":"26_CR21"},{"unstructured":"Neves, M., et al.: Overview of task 1 in CLEF eHealth 2019: indexing German non-technical summaries of animal experiments. In: CLEF 2019 Online Working Notes. CEUR-WS (2019)","key":"26_CR22"},{"unstructured":"Palotti, J., et al.: CLEF eHealth evaluation lab 2015, Task 2: retrieving information about medical symptoms. In: CLEF 2015 Online Working Notes. CEUR-WS (2015)","key":"26_CR23"},{"issue":"1","key":"26_CR24","doi-asserted-by":"publisher","first-page":"e10986","DOI":"10.2196\/10986","volume":"21","author":"J Palotti","year":"2019","unstructured":"Palotti, J., Zuccon, G., Hanbury, A.: Consumer health search on the web: study of web page understandability and its integration in ranking algorithms. J. Med. Internet Res. 21(1), e10986 (2019). \n                      https:\/\/doi.org\/10.2196\/10986","journal-title":"J. Med. Internet Res."},{"unstructured":"Palotti, J., et al.: CLEF 2017 task overview: the IR task at the ehealth evaluation lab. In: Working Notes of Conference and Labs of the Evaluation (CLEF) Forum, CEUR Workshop Proceedings (2017)","key":"26_CR25"},{"unstructured":"Suominen, H.: CLEFeHealth2012 \u2013 The CLEF 2012 workshop on cross-language evaluation of methods, applications, and resources for eHealth document analysis. In: Forner, P., Karlgren, J., Womser-Hacker, C., Ferro, N. (eds.) CLEF 2012 Working Notes. CEUR Workshop Proceedings. CEUR-WS.org, ISSN 1613\u20130073 (2012). \n                      http:\/\/ceur-ws.org\/Vol-1178\/","key":"26_CR26"},{"issue":"7","key":"26_CR27","doi-asserted-by":"publisher","first-page":"e10961","DOI":"10.2196\/10961","volume":"7","author":"H Suominen","year":"2018","unstructured":"Suominen, H., Kelly, L., Goeuriot, L.: Scholarly influence of the conference and labs of the evaluation forum eHealth initiative: review and bibliometric study of the 2012 to 2017 outcomes. JMIR Res. Protoc. 7(7), e10961 (2018)","journal-title":"JMIR Res. Protoc."},{"key":"26_CR28","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"286","DOI":"10.1007\/978-3-319-98932-7_26","volume-title":"Experimental IR Meets Multilinguality, Multimodality, and Interaction","author":"H Suominen","year":"2018","unstructured":"Suominen, H., et al.: Overview of the CLEF eHealth evaluation lab 2018. In: Bellot, P., et al. (eds.) Experimental IR Meets Multilinguality, Multimodality, and Interaction. LNCS, vol. 11018, pp. 286\u2013301. Springer, Cham (2018). \n                      https:\/\/doi.org\/10.1007\/978-3-319-98932-7_26"},{"key":"26_CR29","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"212","DOI":"10.1007\/978-3-642-40802-1_24","volume-title":"Information Access Evaluation. Multilinguality, Multimodality, and Visualization","author":"H Suominen","year":"2013","unstructured":"Suominen, H., et al.: Overview of the ShARe\/CLEF eHealth evaluation lab 2013. In: Forner, P., M\u00fcller, H., Paredes, R., Rosso, P., Stein, B. (eds.) CLEF 2013. LNCS, vol. 8138, pp. 212\u2013231. Springer, Heidelberg (2013). \n                      https:\/\/doi.org\/10.1007\/978-3-642-40802-1_24"},{"unstructured":"Zuccon, G., et al.: The IR task at the CLEF eHealth evaluation lab 2016: user-centred health information retrieval. In: CLEF 2016 Evaluation Labs and Workshop: Online Working Notes. CEUR-WS, September 2016","key":"26_CR30"}],"container-title":["Lecture Notes in Computer Science","Experimental IR Meets Multilinguality, Multimodality, and Interaction"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-28577-7_26","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,9,1]],"date-time":"2019-09-01T19:20:27Z","timestamp":1567365627000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-28577-7_26"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030285760","9783030285777"],"references-count":30,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-28577-7_26","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"3 August 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CLEF","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference of the Cross-Language Evaluation Forum for European Languages","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Lugano","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Switzerland","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"9 September 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"12 September 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"clef2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/clef2019.clef-initiative.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"30","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"7","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"8","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"23% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}