{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:00:53Z","timestamp":1755838853581},"publisher-location":"Cham","reference-count":49,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030229474"},{"type":"electronic","value":"9783030229481"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-22948-1_6","type":"book-chapter","created":{"date-parts":[[2019,8,13]],"date-time":"2019-08-13T05:02:59Z","timestamp":1565672579000},"page":"161-173","update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["EaaS: Evaluation-as-a-Service and Experiences from the VISCERAL Project"],"prefix":"10.1007","author":[{"given":"Henning","family":"M\u00fcller","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2019,8,14]]},"reference":[{"key":"6_CR1","first-page":"88","volume":"7488","author":"M Agosti","year":"2012","unstructured":"Agosti M, Di Buccio E, Ferro N, Masiero I, Peruzzo S, Silvello G (2012) Directions: design and specification of an ir evaluation infrastructure. In: Springer (ed) Multilingual and multimodal information access evaluation\u2014third international conference of the cross-language evaluation forum, LNCS, vol 7488, pp 88\u201399","journal-title":"LNCS, vol"},{"key":"6_CR2","doi-asserted-by":"crossref","unstructured":"Armstrong TG, Moffat A, Webber W, Zobel J (2009a) EvaluatIR: an online tool for evaluating and comparing ir systems. In: Proceedings of the 32nd international ACM SIGIR conference, SIGIR\u201909. ACM, New York, p 833. http:\/\/doi.acm.org\/10.1145\/1571941.1572153","DOI":"10.1145\/1571941.1572153"},{"key":"6_CR3","doi-asserted-by":"publisher","first-page":"601","DOI":"10.1145\/1645953.1646031","volume-title":"Proceeding of the 18th ACM conference on Information and knowledge management, CIKM\u201909","author":"TG Armstrong","year":"2009","unstructured":"Armstrong TG, Moffat A, Webber W, Zobel J (2009b) Improvements that don\u2019t add up: ad-hoc retrieval results since 1998. In Proceeding of the 18th ACM conference on Information and knowledge management, CIKM\u201909. ACM, New York, pp 601\u2013610. http:\/\/doi.acm.org\/10.1145\/1645953.1646031"},{"key":"6_CR4","unstructured":"Blanco R, Zaragoza H (2011) Beware of relatively large but meaningless improvements. Tech. rep., Yahoo Research"},{"key":"6_CR5","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1108\/EUM0000000007198","volume":"53","author":"P Borlund","year":"1997","unstructured":"Borlund P, Ingwersen P (1997) The development of a method for the evaluation of interactive information retrieval systems. J Doc 53:225\u2013250","journal-title":"J Doc"},{"key":"6_CR6","first-page":"78","volume":"3","author":"M Braschler","year":"2002","unstructured":"Braschler M, Peters C (2002) The CLEF campaigns: evaluation of cross\u2013language information retrieval systems. CEPIS UPGRADE III 3:78\u201381","journal-title":"CEPIS UPGRADE III"},{"key":"6_CR7","unstructured":"Cleverdon CW (1962) Report on the testing and analysis of an investigation into the comparative efficiency of indexing systems. Tech. rep., Aslib Cranfield Research Project, Cranfield"},{"key":"6_CR8","unstructured":"Cleverdon C, Mills J, Keen M (1966) Factors determining the performance of indexing systems. Tech. rep., ASLIB Cranfield Research Project, Cranfield"},{"key":"6_CR9","first-page":"248","volume":"2009","author":"J Deng","year":"2009","unstructured":"Deng J, Dong W, Socher R, Li LJ, Li K, Fei-Fei L (2009) ImageNet: a large\u2013scale hierarchical image database. In: IEEE conference on computer vision and pattern recognition, CVPR 2009, pp 248\u2013255","journal-title":"IEEE conference on computer vision and pattern recognition, CVPR"},{"key":"6_CR10","first-page":"483","volume-title":"International conference on theory and practice of digital libraries","author":"GM Nunzio Di","year":"2005","unstructured":"Di Nunzio GM, Ferro N (2005) Direct: a system for evaluating information access components of digital libraries. In: International conference on theory and practice of digital libraries. Springer, Berlin, pp 483\u2013484"},{"key":"6_CR11","doi-asserted-by":"crossref","unstructured":"Forsyth DA (2002) Benchmarks for storage and retrieval in multimedia databases. In: SPIE Proceedings of storage and retrieval for media databases, vol 4676, San Jose, pp 240\u2013247 (SPIE photonics west conference)","DOI":"10.1117\/12.451112"},{"key":"6_CR12","doi-asserted-by":"publisher","first-page":"c6815","DOI":"10.1136\/bmj.c6815","volume":"341","author":"AG Fraser","year":"2010","unstructured":"Fraser AG, Dunstan FD (2010) On the impossibility of being expert. BMJ 341:c6815","journal-title":"BMJ"},{"key":"6_CR13","doi-asserted-by":"crossref","first-page":"1125","DOI":"10.1145\/2348283.2348501","volume-title":"Proceedings of the 35th international ACM SIGIR conference on research and development in information retrieval","author":"T Gollub","year":"2012","unstructured":"Gollub T, Stein B, Burrows S (2012) Ousting ivory tower research: towards a web framework for providing experiments as a service. In: Proceedings of the 35th international ACM SIGIR conference on research and development in information retrieval. ACM, New York, pp 1125\u20131126"},{"key":"6_CR14","doi-asserted-by":"crossref","unstructured":"Gonzalo J, Clough P, Vallin A (2005) Overview of the CLEF 2005 interactive track. In: Working notes of the 2005 CLEF workshop, Vienna","DOI":"10.1007\/11878773_30"},{"key":"6_CR15","doi-asserted-by":"publisher","first-page":"124","DOI":"10.1007\/978-3-642-15998-5_14","volume-title":"Multilingual and Multimodal Information Access Evaluation","author":"Allan Hanbury","year":"2010","unstructured":"Hanbury A, M\u00fcller H (2010) Automated component\u2013level evaluation: present and future. In: International conference of the cross\u2013language evaluation forum (CLEF). Lecture notes in computer science (LNCS), vol 6360. Springer, Berlin, pp 124\u2013135"},{"key":"6_CR16","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1007\/978-3-642-33247-0_3","volume-title":"Information Access Evaluation. Multilinguality, Multimodality, and Visual Analytics","author":"Allan Hanbury","year":"2012","unstructured":"Hanbury A, M\u00fcller H, Langs G, Weber MA, Menze BH, Fernandez TS (2012) Bringing the algorithms to the data: cloud\u2013based benchmarking for medical image analysis. In: CLEF conference. Lecture notes in computer science. Springer, Berlin"},{"key":"6_CR17","unstructured":"Hanbury A, M\u00fcller H, Balog K, Brodt T, Cormack GV, Eggel I, Gollub T, Hopfgartner F, Kalpathy-Cramer J, Kando N, Krithara A, Lin J, Mercer S, Potthast M (2015) Evaluation\u2013as\u2013a\u2013service: overview and outlook. arXiv 1512.07454"},{"volume-title":"Cloud\u2013based benchmarking of medical image analysis","year":"2017","key":"6_CR18","unstructured":"Hanbury A, M\u00fcller H, Langs G (eds) (2017) Cloud\u2013based benchmarking of medical image analysis. Springer, Berlin"},{"key":"6_CR19","unstructured":"Harman D (1992) Overview of the first text REtrieval conference (TREC-1). In: Proceedings of the first text REtrieval conference (TREC\u20131), Washington, pp 1\u201320"},{"key":"6_CR20","first-page":"250","volume-title":"International conference of the cross-language evaluation forum for European languages","author":"F Hopfgartner","year":"2014","unstructured":"Hopfgartner F, Kille B, Lommatzsch A, Plumbaum T, Brodt T, Heintz T (2014) Benchmarking news recommendations in a living lab. In: International conference of the cross-language evaluation forum for European languages. Springer, Berlin, pp 250\u2013267"},{"issue":"1","key":"6_CR21","doi-asserted-by":"publisher","first-page":"57","DOI":"10.1145\/2795403.2795416","volume":"49","author":"F Hopfgartner","year":"2015","unstructured":"Hopfgartner F, Hanbury A, M\u00fcller H, Kando N, Mercer S, Kalpathy-Cramer J, Potthast M, Gollub T, Krithara A, Lin J, Balog K, Eggel I (2015) Report on the evaluation-as-a-service (EAAS) expert workshop. ACM SIGIR Forum 49(1):57\u201365","journal-title":"ACM SIGIR Forum"},{"issue":"4","key":"6_CR22","first-page":"15","volume":"10","author":"F Hopfgartner","year":"2018","unstructured":"Hopfgartner F, Hanbury A, M\u00fcller H, Balog K, Brodt T, Cormack GV, Eggel I, Gollub T, Kalpathy-Cramer J, Kando N, Krithara A, Lin J, Mercer S, Potthast M (2018) Evaluation-as-a-service in the computational sciences: overview and outlook. J Data Inf Qual 10(4):15","journal-title":"J Data Inf Qual"},{"issue":"8","key":"6_CR23","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124","volume":"2","author":"JP Ioannidis","year":"2005","unstructured":"Ioannidis JP (2005) Why most published research findings are false. PLoS Med 2(8):e124","journal-title":"PLoS Med"},{"issue":"11","key":"6_CR24","doi-asserted-by":"publisher","first-page":"2459","DOI":"10.1109\/TMI.2016.2578680","volume":"35","author":"O Jimenez-del-Toro","year":"2016","unstructured":"Jimenez-del-Toro O, M\u00fcller H, Krenn M, Gruenberg K, Taha AA, Winterstein M, Eggel I, Foncubierta-Rodr\u00edguez A, Goksel O, Jakab A, Kontokotsios G, Langs G, Menze B, Salas Fernandez T, Schaer R, Walleyo A, Weber MA, Dicente Cid Y, Gass T, Heinrich M, Jia F, Kahl F, Kechichian R, Mai D, Spanier AB, Vincent G, Wang C, Wyeth D, Hanbury A (2016) Cloud\u2013based evaluation of anatomical structure segmentation and landmark detection algorithms: VISCERAL anatomy benchmarks. IEEE Trans Med Imaging 35(11):2459\u20132475","journal-title":"IEEE Trans Med Imaging"},{"key":"6_CR25","unstructured":"Jones KS, van Rijsbergen C (1975) Report on the need for and provision of an ideal information retrieval test collection. British Library Research and Development Report 5266, Computer Laboratory, University of Cambridge"},{"key":"6_CR26","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1016\/j.compmedimag.2014.03.004","volume":"39","author":"Jayashree Kalpathy-Cramer","year":"2015","unstructured":"Kalpathy-Cramer J, Garc\u00eda Seco de Herrera A, Demner-Fushman D, Antani S, Bedrick S, M\u00fcller H (2015) Evaluating performance of biomedical image retrieval systems: overview of the medical image retrieval task at ImageCLEF 2004\u20132014. Comput Med Imaging Graph 39:55\u201361","journal-title":"Computerized Medical Imaging and Graphics"},{"key":"6_CR27","first-page":"103","volume-title":"Medical computer vision: algorithms for big data: international workshop, MCV 2015, Held in Conjunction with MICCAI 2015, Munich, Germany, October 9, 2015","author":"M Krenn","year":"2016","unstructured":"Krenn M, Dorfer M, Jimenez-del-Toro O, M\u00fcller H, Menze B, Weber MA, Hanbury A, Langs G (2016) Creating a large\u2013scale silver corpus from multiple algorithmic segmentations. In: Menze B, Langs G, Montillo A, Kelm M, M\u00fcller H, Zhang S, Cai W, Metaxas D (eds) Medical computer vision: algorithms for big data: international workshop, MCV 2015, Held in Conjunction with MICCAI 2015, Munich, Germany, October 9, 2015, Revised Selected Papers, Springer International Publishing, pp 103\u2013115"},{"key":"6_CR28","unstructured":"Krizhevsky A, Sutskever I, Hinton GE (2012) ImageNet classification with deep convolutional neural networks. In: Pereira F, Burges CJC, Bottou L, Weinberger KQ (eds) Advances in neural information processing systems, vol 25, Curran Associates, pp 1097\u20131105"},{"key":"6_CR29","first-page":"92","volume-title":"Medical content-based retrieval for clinical decision support","author":"G Langs","year":"2012","unstructured":"Langs G, Hanbury A, Menze B, M\u00fcller H (2012) VISCERAL: towards large data in medical imaging\u2014challenges and directions. In: Greenspan H, M\u00fcller H, Syeda-Mahmood T (eds) Medical content-based retrieval for clinical decision support. Lecture notes in computer science. Springer, Berlin, pp 92\u201398"},{"issue":"6","key":"6_CR30","doi-asserted-by":"publisher","first-page":"539","DOI":"10.3414\/ME11-02-0025","volume":"51","author":"D Markonis","year":"2012","unstructured":"Markonis D, Holzer M, Dungs S, Vargas A, Langs G, Kriewel S, M\u00fcller H (2012) A survey on visual information search behavior and requirements of radiologists. Methods Inf Med 51(6):539\u2013548","journal-title":"Methods Inf Med"},{"key":"6_CR31","first-page":"224","volume":"192","author":"D Markonis","year":"2013","unstructured":"Markonis D, Baroz F, Ruiz de Castaneda RL, Boyer C, M\u00fcller H (2013) User tests for assessing a medical image retrieval system: a pilot study. Stud Health Technol Inform 192:224\u2013228","journal-title":"Stud Health Technol Inform"},{"issue":"6","key":"6_CR32","doi-asserted-by":"publisher","first-page":"1341","DOI":"10.1002\/asi.23721","volume":"68","author":"MS Mayernik","year":"2017","unstructured":"Mayernik MS, Hart DL, Mauli DL, Weber NM (2017) Assessing and tracing the outcomes and impact of research infrastructures. J Am Soc Inf Sci Technol 68(6):1341\u20131359","journal-title":"J Am Soc Inf Sci Technol"},{"key":"6_CR33","doi-asserted-by":"crossref","unstructured":"M\u00fcller H, M\u00fcller W, Marchand-Maillet S, Squire DM, Pun T (2001) Automated benchmarking in content\u2013based image retrieval. In: Proceedings of the second international conference on multimedia and exposition (ICME\u20192001). IEEE Computer Society, Silver Spring, pp 321\u2013324","DOI":"10.1109\/ICME.2001.1237929"},{"key":"6_CR34","series-title":"Lecture notes in computer science (LNCS), vol","doi-asserted-by":"crossref","first-page":"38","DOI":"10.1007\/3-540-45479-9_5","volume-title":"Proceedings of the international conference on the challenge of image and video retrieval (CIVR 2002)","author":"H M\u00fcller","year":"2002","unstructured":"M\u00fcller H, Marchand-Maillet S, Pun T (2002) The truth about corel\u2013evaluation in image retrieval. In: Lew MS, Sebe N, Eakins JP (eds) Proceedings of the international conference on the challenge of image and video retrieval (CIVR 2002). Lecture notes in computer science (LNCS), vol 2383. Springer, Berlin, pp 38\u201349"},{"issue":"Pt 2","key":"6_CR35","first-page":"1319","volume":"129","author":"H M\u00fcller","year":"2007","unstructured":"M\u00fcller H, Boyer C, Gaudinat A, Hersh W, Geissbuhler A (2007) Analyzing web log files of the Health On the Net HONmedia search engine to define typical image search tasks for image retrieval evaluation. Stud Health Technol Inform 129(Pt 2):1319\u20131323","journal-title":"Stud Health Technol Inform"},{"key":"6_CR36","doi-asserted-by":"crossref","unstructured":"M\u00fcller H, Clough P, Deselaers T, Caputo B (eds) (2010) ImageCLEF\u2013experimental evaluation in visual information retrieval. In: The Springer international series on information retrieval, vol 32. Springer, Berlin","DOI":"10.1007\/978-3-642-15181-1"},{"issue":"1","key":"6_CR37","first-page":"35","volume":"51","author":"H M\u00fcller","year":"2016","unstructured":"M\u00fcller H, Kalpathy-Cramer J, Hanbury A, Farahani K, Sergeev R, Paik JH, Klein A, Criminisi A, Trister A, Norman T, Kennedy D, Srinivasa G, Mamonov A, Preuss N (2016) Report on the cloud\u2013based evaluation approaches workshop 2015. ACM SIGIR Forum 51(1):35\u201341","journal-title":"ACM SIGIR Forum"},{"issue":"6","key":"6_CR38","first-page":"161","volume":"7","author":"KE Niemeyer","year":"2016","unstructured":"Niemeyer KE, Smith AM, Katz DS (2016) The challenge and promise of software citation for credit, identification, discovery, and reuse. J Data Inform Qual 7(6):161\u2013165","journal-title":"J Data Inform Qual"},{"key":"6_CR39","unstructured":"Ounis I, Macdonald C, Lin J, Soboroff I (2011) Overview of the trec-2011 microblog track. In: Proceedings of the 20th text REtrieval conference (TREC 2011), vol 32"},{"key":"6_CR40","unstructured":"Rowe BR, Wood DW, Link AN, Simoni DA (2010) Economic impact assessment of NIST text retrieval conference (TREC) program. Technical report project number 0211875, National Institute of Standards and Technology"},{"key":"6_CR41","unstructured":"Salton G (1971) The SMART retrieval system, experiments in automatic document processing. Prentice Hall, Englewood Cliffs"},{"key":"6_CR42","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1002\/asi.23917","volume":"69","author":"G Silvello","year":"2018","unstructured":"Silvello G (2018) Theory and practice of data citation. J Assoc Inf Sci Technol 69:6\u201320","journal-title":"J Assoc Inf Sci Technol"},{"issue":"2","key":"6_CR43","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s00799-016-0172-8","volume":"18","author":"G Silvello","year":"2017","unstructured":"Silvello G, Bordea G, Ferro N, Buitelaar P, Bogers T (2017) Semantic representation and enrichment of information retrieval experimental data. Int J Digit Libr 18(2):145\u2013172","journal-title":"Int J Digit Libr"},{"key":"6_CR44","doi-asserted-by":"crossref","unstructured":"Smeaton AF, Kraaij W, Over P (2003) TRECVID 2003: an overview. In: Proceedings of the TRECVID 2003 conference","DOI":"10.1145\/1027527.1027678"},{"issue":"4","key":"6_CR45","doi-asserted-by":"publisher","first-page":"613","DOI":"10.1002\/asi.21494","volume":"62","author":"CV Thornley","year":"2011","unstructured":"Thornley CV, Johnson AC, Smeaton AF, Lee H (2011) The scholarly impact of TRECVid (2003\u20132009). J Am Soc Inf Sci Technol 62(4):613\u2013627","journal-title":"J Am Soc Inf Sci Technol"},{"issue":"11","key":"6_CR46","doi-asserted-by":"publisher","first-page":"1463","DOI":"10.1001\/jamaoncol.2017.0473","volume":"3","author":"AD Trister","year":"2017","unstructured":"Trister AD, Buist DS, Lee CI (2017) Will machine learning tip the balance in breast cancer screening? JAMA Oncol 3(11):1463\u20131464","journal-title":"JAMA Oncol"},{"issue":"1","key":"6_CR47","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1186\/s12859-015-0564-6","volume":"16","author":"G Tsatsaronis","year":"2015","unstructured":"Tsatsaronis G, Balikas G, Malakasiotis P, Partalas I, Zschunke M, Alvers MR, Weissenborn D, Krithara A, Petridis S, Polychronopoulos D, et al (2015) An overview of the bioasq large-scale biomedical semantic indexing and question answering competition. BMC Bioinf 16(1):138","journal-title":"BMC Bioinf"},{"key":"6_CR48","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1007\/978-3-642-23708-9_12","volume-title":"Multilingual and Multimodal Information Access Evaluation","author":"Theodora Tsikrika","year":"2011","unstructured":"Tsikrika T, Garc\u00eda Seco de Herrera A, M\u00fcller H (2011) Assessing the scholarly impact of ImageCLEF. In: CLEF 2011. Lecture notes in computer science (LNCS). Springer, Berlin, pp 95\u2013106"},{"key":"6_CR49","first-page":"1","volume-title":"Lecture Notes in Computer Science","author":"Theodora Tsikrika","year":"2013","unstructured":"Tsikrika T, Larsen B, M\u00fcller H, Endrullis S, Rahm E (2013) The scholarly impact of CLEF (2000\u20132009). In: information access evaluation, multilinguality, multimodality, and visualization. Springer, Berlin, pp 1\u201312"}],"container-title":["The Information Retrieval Series","Information Retrieval Evaluation in a Changing World"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-22948-1_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,19]],"date-time":"2023-09-19T01:23:08Z","timestamp":1695086588000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-22948-1_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030229474","9783030229481"],"references-count":49,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-22948-1_6","relation":{},"ISSN":["1387-5264"],"issn-type":[{"type":"print","value":"1387-5264"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"14 August 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}