{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,5,27]],"date-time":"2025-05-27T04:07:44Z","timestamp":1748318864625,"version":"3.41.0"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2003,9,1]],"date-time":"2003-09-01T00:00:00Z","timestamp":1062374400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2003,9,1]],"date-time":"2003-09-01T00:00:00Z","timestamp":1062374400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Multimedia Tools and Applications"],"published-print":{"date-parts":[[2003,9]]},"DOI":"10.1023\/a:1025034215859","type":"journal-article","created":{"date-parts":[[2003,9,16]],"date-time":"2003-09-16T22:24:47Z","timestamp":1063751087000},"page":"55-73","source":"Crossref","is-referenced-by-count":11,"title":["A Framework for Benchmarking in CBIR"],"prefix":"10.1007","volume":"21","author":[{"given":"Henning","family":"M\u00fcller","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wolfgang","family":"M\u00fcller","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"St\u00e9phane","family":"Marchand-Maillet","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thierry","family":"Pun","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David McG.","family":"Squire","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"5140132_CR1","unstructured":"G. Beretta and R. Schettini (Eds.), \u201cInternet Imaging III,\u201d in SPIE Proceedings. San Jose: California, USA, 2002, Vol. 4672, (SPIE Photonics West Conference)."},{"key":"5140132_CR2","doi-asserted-by":"crossref","first-page":"225","DOI":"10.1108\/EUM0000000007198","volume":"53","author":"P. Borlund","year":"1997","unstructured":"P. Borlund and P. Ingwersen, \u201cThe development of a method for the evaluation of interactive information retrieval systems,\u201d Journal of Documentation, Vol. 53, pp. 225\u2013250, 1997.","journal-title":"Journal of Documentation"},{"key":"5140132_CR3","unstructured":"C.W. Cleverdon, \u201cReport on the testing and analysis of an investigation into the comparative efficiency pf indexing systems,\u201d Technical Report, Aslib Cranfield Research Project, Cranfield, USA, 1962."},{"key":"5140132_CR4","unstructured":"C.W. Cleverdon, L. Mills, and M. Keen, \u201cFactors determining the performance of indexing systems,\u201d Technical Report, ASLIB Cranfield Research Project, Cranfield, 1966."},{"key":"5140132_CR5","first-page":"66","volume-title":"Advances in Digital Libraries (ADL'96)","author":"I.J. Cox","year":"1996","unstructured":"I.J. Cox, M.L. Miller, S.M. Omohundro, and P.N. Yianilos, \u201cTarget testing and the PicHunter Bayesian multimedia retrieval system,\u201d in Advances in Digital Libraries (ADL'96). Library of Congress, Washington, D.C., 1996, pp. 66\u201375."},{"key":"5140132_CR6","volume-title":"Third International Conference on Visual Information Systems (VISUAL'99)","author":"A. Dimai","year":"1999","unstructured":"A. Dimai, \u201cAssessment of effectiveness of content-based image retrieval systems,\u201d in Third International Conference on Visual Information Systems (VISUAL'99), D.P. Huijsmans and A.W.M. Smeulders (Eds.), Springer-Verlag: Amsterdam, The Netherlands, 1999."},{"key":"5140132_CR7","unstructured":"J.G. Dy, C.E. Brodley, A. Kak, C.-R. Shyu, and L.S. Broderick, \u201cThe customized-queries approach to CBIR using EM,\u201d in Proceedings of the 1999 IEEE Conference on Computer Vision and Pattern Recognition (CVPR'99). Fort Collins, Colorado, USA, 1999, pp. 400\u2013406."},{"key":"5140132_CR8","doi-asserted-by":"crossref","unstructured":"J.P. Eakins, B.J.M., and M.E. Graham, \u201cSimilarity retrieval of trademark images,\u201d IEEE Multimedia Magazine April June, 53\u201363, 1998.","DOI":"10.1109\/93.682526"},{"key":"5140132_CR9","doi-asserted-by":"crossref","unstructured":"N.J. Gunther and G. Beretta, \u201cA benchmark for image retrieval using distributed systems over the internet: BIRDS-I,\u201d Technical Report, HP Labs, Palo Alto, Technical Report HPL-2000-162, San Jose, 2001.","DOI":"10.1117\/12.411898"},{"key":"5140132_CR10","doi-asserted-by":"crossref","unstructured":"D. Harman, \u201cOverview of the First Text REtrieval Conference (TREC-1),\u201d in Proceedings of the First Text REtrieval Conference (TREC-1), Washington DC, USA, 1992, pp. 1\u201320.","DOI":"10.6028\/NIST.SP.500-207"},{"key":"5140132_CR11","series-title":"Lecture Notes in Computer Science","volume-title":"Third international conference on visual information systems (VISUAL'99)","year":"1999","unstructured":"D.P. Huijsmans and A.W.M. Smeulders (Eds.), \u201cThird international conference on visual information systems (VISUAL'99),\u201d No. 1614 in Lecture Notes in Computer Science. Springer-Verlag: Amsterdam, The Netherlands, 1999."},{"key":"5140132_CR12","volume-title":"Proceedings of the Second International Conference on Multimedia and Exposition (ICME'2001)","author":"ICME'2001","year":"2001","unstructured":"ICME'2001, \u201cProceedings of the Second International Conference on Multimedia and Exposition (ICME'2001),\u201d Tokyo, Japan, IEEE, 2001."},{"key":"5140132_CR13","unstructured":"C. J\u00f6rgensen, \u201cClassifying images: Criteria for grouping as revealed in a sorting task,\u201d in Proceedings of the6th ASIS SIG\/CR Classification Research Workshop, Chicago, IL, USA, 1995, pp. 65\u201378."},{"key":"5140132_CR14","doi-asserted-by":"crossref","unstructured":"C. J\u00f6rgensen and P. J\u00f6rgensen, \u201cTesting a vocabulary for image indexing and ground truthing,\u201d in SPIE Proceedings, G. Beretta and R. Schettini (Eds.), San Jose: California, USA, 2002, Vol. 4672, (SPIE Photonics West Conference).","DOI":"10.1117\/12.452674"},{"key":"5140132_CR15","doi-asserted-by":"crossref","unstructured":"M. Koskela, J. Laaksonen, S. Laakso, and E. Oja, \u201cEvaluating the performance of content-based image retrieval systems,\u201d in Fourth International Conference on Visual Information Systems (VISUAL'2000), R. Laurini (Ed.), Springer-Verlag, 2000.","DOI":"10.1007\/3-540-40053-2_38"},{"key":"5140132_CR16","series-title":"Lecture Notes in Computer Science","volume-title":"Fourth International Conference on Visual Information Systems (VISUAL'2000)","year":"2000","unstructured":"R. Laurini (Ed.), \u201cFourth International Conference on Visual Information Systems (VISUAL'2000),\u201d No. 1929 in Lecture Notes in Computer Science, Springer-Verlag: Lyon, France, 2000."},{"key":"5140132_CR17","doi-asserted-by":"crossref","unstructured":"C. Leung and H. Ip, \u201cBenchmarking for content-based visual information search,\u201d in Fourth International Conference on Visual Information Systems (VISUAL'2000), R. Laurini (Ed.), Springer-Verlag, 2000.","DOI":"10.1007\/3-540-40053-2_39"},{"key":"5140132_CR18","doi-asserted-by":"crossref","unstructured":"M. Markkula and E. Sormunen, \u201c\u2018Searching for photos\u2014Journalists\u2019 practices in pictorial IR,\u201d in The Challenge of Image Retrieval, A Workshop and Symposium on Image Retrieval, J.P. Eakins, D.J. Harper, and J. Jose (Eds.), Newcastle upon Tyne, The British Computer Society, 1998.","DOI":"10.14236\/ewic\/CIR1998.8"},{"key":"5140132_CR19","first-page":"35","volume-title":"Image Databases and Multi-Media Search","author":"F. Mokhtarian","year":"1996","unstructured":"F. Mokhtarian, S. Abbasi, and J. Kittler, \u201cEfficient and robust retrieval by shape content through curvature scale space,\u201d in Image Databases and Multi-Media Search, A.W.M. Smeulders and R. Jain (Eds.), Amsterdam University Press: Amsterdam, The Netherlands, 1996, pp. 35\u201342."},{"key":"5140132_CR20","unstructured":"MPEG Requirements Group, \u201cMPEG-7: Context and objectives (version 10 Atlantic City),\u201d Doc. ISO\/IEC JTC1\/SC29\/WG11, International Organisation for Standardisation, 1998."},{"key":"5140132_CR21","volume-title":"Proceedings of the Second International Conference on Multimedia and Exposition (ICME'2001)","author":"H. M\u00fcller","year":"2001","unstructured":"H. M\u00fcller, W. M\u00fcller, S. Marchand-Maillet, D.M. Squire, and T. Pun, \u201cAutomated benchmarking in contentbased image retrieval,\u201d in Proceedings of the Second International Conference on Multimedia and Exposition (ICME'2001),\u201d Tokyo, Japan, IEEE, 2001a."},{"key":"5140132_CR22","doi-asserted-by":"crossref","first-page":"50","DOI":"10.1145\/500933.500949","volume-title":"Proceedings of the ACMMultimediaWorkshop on Multimedia Information Retrieval (ACM MIR 2001)","author":"H. M\u00fcller","year":"2001","unstructured":"H. M\u00fcller, W. M\u00fcller, S. Marchand-Maillet, D.M. Squire, and T. Pun, \u201cA web-based evaluation system for content-based image retrieval,\u201d in Proceedings of the ACMMultimediaWorkshop on Multimedia Information Retrieval (ACM MIR 2001). The Association for Computing Machinery: Ottawa, Canada, 2001b, pp. 50\u201354."},{"key":"5140132_CR23","doi-asserted-by":"crossref","unstructured":"H. M\u00fcller, W. M\u00fcller, D.M. Squire, S. Marchand-Maillet, and T. Pun, \u201cPerformance evaluation in contentbased image retrieval: Overview and proposals,\u201d Pattern Recognition Letters, Vol. 22, No. 5, 2001c.","DOI":"10.1016\/S0167-8655(00)00118-5"},{"key":"5140132_CR24","unstructured":"W. M\u00fcller, S. Marchand-Maillet, H. M\u00fcller, and T. Pun, \u201cTowards a fair benchmark for image browsers,\u201d in SPIE Photonics East, Voice, Video, and Data Communications. Boston, MA, USA, 2000."},{"key":"5140132_CR25","unstructured":"W. M\u00fcller, Z. Pe\u010denovi\u0107, A.P. de Vries, D.M. Squire, H. M\u00fcller, and T. Pun, \u201cMRML: Towards an extensible standard for multimedia querying and benchmarking\u2014Draft proposal,\u201d Technical Report 99.04, ComputerVision Group, Computing Centre, University of Geneva, rue G\u00e9n\u00e9ral Dufour, 24, CH-1211 Gen\u00e8ve, Switzerland, 1999."},{"key":"5140132_CR26","first-page":"45","volume-title":"Proceedings of the Second International Conference on Multimedia and Exposition (ICME'2001)","author":"M. Nakazato","year":"2001","unstructured":"M. Nakazato and T.S. Huang, \u201c3D Mars: Immersive virtual reality for content-based image retrieval,\u201d in Proceedings of the Second International Conference on Multimedia and Exposition (ICME'2001),\u201d Tokyo, Japan, IEEE, 2001, pp. 45\u201348."},{"key":"5140132_CR27","doi-asserted-by":"crossref","first-page":"333","DOI":"10.1023\/A:1009641123797","volume":"4","author":"A.D. Narasimhalu","year":"1997","unstructured":"A.D. Narasimhalu, M.S. Kankanhalli, and J. Wu, \u201cBenchmarking multimedia databases,\u201d Multimedia Tools and Applications, Vol. 4, pp. 333\u2013356, 1997.","journal-title":"Multimedia Tools and Applications"},{"key":"5140132_CR28","doi-asserted-by":"crossref","unstructured":"T. Pfund and S. Marchand-Maillet, \u201cDynamic multimedia annotation tool,\u201d in SPIE Proceedings, G. Beretta and R. Schettini (Eds.), San Jose: California, USA, 2002, Vol. 4672, (SPIE Photonics West Conference).","DOI":"10.1117\/12.452675"},{"key":"5140132_CR29","unstructured":"P.S. Salembier and B.S. Manjunath, \u201cAudiovisual content description and retrieval: Tools and MPEG-7 standardization techniques,\u201d in IEEE Internation Conference on Image Processing (ICIP 2000). Vancouver, BC, Canada, 2000."},{"key":"5140132_CR30","volume-title":"The SMART Retrieval System, Experiments in Automatic Document Processing","author":"G. Salton","year":"1971","unstructured":"G. Salton, The SMART Retrieval System, Experiments in Automatic Document Processing. Prentice Hall: Englewood Cliffs, NJ, USA, 1971."},{"key":"5140132_CR31","unstructured":"C.-R. Shyu, A. Kak, C. Brodley, and L.S. Broderick, \u201cTesting for human perceptual categories in a physician-in-the-loop CBIR system for medical imagery,\u201d in IEEE Workshop on Content-Based Access of Image and Video Libraries (CBAIVL'99). Fort Collins, CO, USA, 1999, pp. 102\u2013108."},{"key":"5140132_CR32","doi-asserted-by":"crossref","unstructured":"J.R. Smith, \u201cImage retrieval evaluation,\u201d in IEEE Workshop on Content-Based Access of Image and Video Libraries (CBAIVL'98). Santa Barbara, CA, USA, 1998, pp. 112\u2013113.","DOI":"10.1109\/IVL.1998.694520"},{"key":"5140132_CR33","unstructured":"K. Sparck Jones and C. van Rijsbergen, \u201cReport on the need for and provision of an ideal information retrieval test collection,\u201d British Library Research and Development Report 5266, Computer Laboratory, University of Cambridge, 1975."},{"key":"5140132_CR34","first-page":"549","volume-title":"Third International Conference on Visual Information Systems (VISUAL'99)","author":"D.M. Squire","year":"1999","unstructured":"D.M. Squire, W. M\u00fcller, and H. M\u00fcller, \u201cRelevance feedback and term weighting schemes for contentbased image retrieval,\u201d in Third International Conference on Visual Information Systems (VISUAL'99), D.P. Huijsmans and A.W.M. Smeulders (Eds.), Springer-Verlag: Amsterdam, The Netherlands, 1999, pp. 549\u2013556."},{"key":"5140132_CR35","first-page":"51","volume-title":"The 10th Scandinavian Conference on Image Analysis (SCIA'97)","author":"D.M. Squire","year":"1997","unstructured":"D.M. Squire and T. Pun, \u201cA comparison of human and machine assessments of image similarity for the organization of image databases,\u201d in The 10th Scandinavian Conference on Image Analysis (SCIA'97), M. Frydrych, J. Parkkinen, and A. Visa (Eds.), Pattern Recognition Society of Finland: Lappeenranta, Finland, 1997, pp. 51\u201358."},{"key":"5140132_CR36","doi-asserted-by":"crossref","unstructured":"E.M. Vorhees and D. Harmann, \u201cOverview of the seventh Text REtrieval Conference (TREC-7),\u201d in The Seventh Text Retrieval Conference. Gaithersburg, MD, USA, 1998, pp. 1\u201323.","DOI":"10.6028\/NIST.SP.500-242.overview"}],"container-title":["Multimedia Tools and Applications"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1023\/A:1025034215859.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1023\/A:1025034215859\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1023\/A:1025034215859.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,26]],"date-time":"2025-05-26T06:32:14Z","timestamp":1748241134000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1023\/A:1025034215859"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2003,9]]},"references-count":36,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2003,9]]}},"alternative-id":["5140132"],"URL":"https:\/\/doi.org\/10.1023\/a:1025034215859","relation":{},"ISSN":["1380-7501","1573-7721"],"issn-type":[{"type":"print","value":"1380-7501"},{"type":"electronic","value":"1573-7721"}],"subject":[],"published":{"date-parts":[[2003,9]]}}}