{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,29]],"date-time":"2025-09-29T08:27:32Z","timestamp":1759134452642},"reference-count":41,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2009,3,19]],"date-time":"2009-03-19T00:00:00Z","timestamp":1237420800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IJDAR"],"published-print":{"date-parts":[[2009,7]]},"DOI":"10.1007\/s10032-009-0083-y","type":"journal-article","created":{"date-parts":[[2009,3,18]],"date-time":"2009-03-18T16:56:35Z","timestamp":1237395395000},"page":"83-96","source":"Crossref","is-referenced-by-count":27,"title":["A performance evaluation protocol for symbol spotting systems in terms of recognition and location indices"],"prefix":"10.1007","volume":"12","author":[{"given":"Mar\u00e7al","family":"Rusi\u00f1ol","sequence":"first","affiliation":[]},{"given":"Josep","family":"Llad\u00f3s","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2009,3,19]]},"reference":[{"key":"83_CR1","doi-asserted-by":"crossref","unstructured":"Aksoy, S., Ye, M., Schauf, M., Song, M., Wang, Y., Haralick, R.: Algorithm performance contest. In: Proceedings of the Fifteenth International Conference on Pattern Recognition, ICPR00, pp. 870\u2013876 (2000). doi: 10.1109\/ICPR.2000.903054","DOI":"10.1109\/ICPR.2000.903054"},{"key":"83_CR2","doi-asserted-by":"crossref","unstructured":"Antonacopoulos, A., Bridson, D.: Performance analysis framework for layout analysis methods. In: Proceedings of the Ninth International Conference on Document Analysis and Recognition, ICDAR07, pp. 1258\u20131262 (2007). doi: 10.1109\/ICDAR.2007.4377117","DOI":"10.1109\/ICDAR.2007.4377117"},{"key":"83_CR3","doi-asserted-by":"crossref","unstructured":"Antonacopoulos, A., Gatos, B., Bridson, D.: ICDAR 2005 page segmentation competition. In: Proceedings of the Eighth International Conference on Document Analysis and Recognition, ICDAR05, pp. 75\u201379 (2005). doi: 10.1109\/ICDAR.2005.184","DOI":"10.1109\/ICDAR.2005.184"},{"key":"83_CR4","unstructured":"Antonacopoulos, A., Gatos, B., Bridson, D.: ICDAR 2007 page segmentation competition. In: Proceedings of the Ninth International Conference on Document Analysis and Recognition, ICDAR07, pp. 1279\u20131283 (2007) doi: 10.1109\/ICDAR.2007.203"},{"key":"83_CR5","doi-asserted-by":"crossref","unstructured":"Antonacopoulos, A., Gatos, B., Karatzas, D.: ICDAR 2003 page segmentation competition. In: Proceedings of the Seventh International Conference on Document Analysis and Recognition, ICDAR03, pp. 688\u2013692 (2003). doi: 10.1109\/ICDAR.2003.1227750","DOI":"10.1109\/ICDAR.2003.1227750"},{"key":"83_CR6","doi-asserted-by":"crossref","unstructured":"Antonacopoulos, A., Karatzas, D., Bridson, D.: Ground truth for layout analysis performance evaluation. In: Document Analysis Systems, DAS06. Lecture Notes on Computer Science, vol. 3872, pp. 302\u2013311. Springer, Berlin (2006). doi: 10.1007\/11669487_27","DOI":"10.1007\/11669487_27"},{"issue":"4","key":"83_CR7","doi-asserted-by":"crossref","first-page":"469","DOI":"10.1145\/235815.235821","volume":"22","author":"C. Barber","year":"1996","unstructured":"Barber C., Dobkin D., Huhdanpaa H.: The quickhull algorithm for convex hulls. ACM Transactions on Mathematical Software 22(4), 469\u2013483 (1996). doi: 10.1145\/235815.235821","journal-title":"ACM Transactions on Mathematical Software"},{"issue":"1","key":"83_CR8","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1002\/(SICI)1097-4571(199401)45:1<12::AID-ASI2>3.0.CO;2-L","volume":"45","author":"M. Buckland","year":"1994","unstructured":"Buckland M., Gey F.: The relationship between recall and precision. J. Am. Soc. Inform. Sci. 45(1), 12\u201319 (1994)","journal-title":"J. Am. Soc. Inform. Sci."},{"key":"83_CR9","doi-asserted-by":"crossref","unstructured":"Davis, J., Goadrich, M.: The relationship between precision\u2013recall and ROC curves. In: Proceedings of the 23rd International Conference on Machine Learning, pp. 233\u2013240 (2006). doi: 10.1145\/1143844.1143874","DOI":"10.1145\/1143844.1143874"},{"key":"83_CR10","doi-asserted-by":"crossref","unstructured":"Delalandre, M., Pridmore, T., Valveny, E., Locteau, H., Trupin, E.: Building synthetic graphical documents for performance evaluation. In: Graphics Recognition. Recent Advances and New Opportunities. Lecture Notes on Computer Science, vol. 5046, pp. 288\u2013298. Springer, Berlin (2008). doi: 10.1007\/978-3-540-88188-9_27","DOI":"10.1007\/978-3-540-88188-9_27"},{"issue":"8","key":"83_CR11","doi-asserted-by":"crossref","first-page":"861","DOI":"10.1016\/j.patrec.2005.10.010","volume":"27","author":"T. Fawcett","year":"2006","unstructured":"Fawcett T.: An introduction to ROC analysis. Pattern Recognit. Lett. 27(8), 861\u2013874 (2006). doi: 10.1016\/j.patrec.2005.10.010","journal-title":"Pattern Recognit. Lett."},{"key":"83_CR12","unstructured":"Holz, F., Witschel, H., Heinrich, G., Heyer, G., Teresniak, S.: An evaluation framework for semantic search in p2p networks. In: Proceedings of the Seventh International Workshop on Innovative Internet Community Systems, I2CS07 (2007)"},{"issue":"3","key":"83_CR13","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1197\/jamia.M1733","volume":"12","author":"G. Hripcsak","year":"2005","unstructured":"Hripcsak G., Rothschild A.: Agreement, the F-measure, and reliability in information retrieval. J. Am. Med. Inform. Assoc 12(3), 296\u2013298 (2005). doi: 10.1016\/j.jamia.2005.01.008","journal-title":"J. Am. Med. Inform. Assoc"},{"key":"83_CR14","first-page":"179","volume":"8","author":"M. Hu","year":"1962","unstructured":"Hu M.: Visual pattern recognition by moment invariants. IRE Trans. Inform. Theory 8, 179\u2013187 (1962)","journal-title":"IRE Trans. Inform. Theory"},{"issue":"2","key":"83_CR15","doi-asserted-by":"crossref","first-page":"245","DOI":"10.1109\/TPAMI.2005.30","volume":"27","author":"D. Huijsmans","year":"2005","unstructured":"Huijsmans D., Sebe N.: How to complete performance graphs in content-based image retrieval: add generality and normalize scope. IEEE Trans. Pattern Anal. Mach. Intell. 27(2), 245\u2013251 (2005). doi: 10.1109\/TPAMI.2005.30","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"83_CR16","doi-asserted-by":"crossref","unstructured":"Kang, B., Kim, H., Lee, S.: Performance analysis of semantic indexing in text retrieval. In: Computational Linguistics and Intelligent Text Processing. Lecture Notes on Computer Science, vol. 2945, pp. 433\u2013436. Springer, Berlin (2004).doi: 10.1007\/b95558","DOI":"10.1007\/b95558"},{"key":"83_CR17","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1109\/34.368168","volume":"17","author":"H. Kauppinen","year":"1995","unstructured":"Kauppinen H., Sepp\u00e4nen T., Pietik\u00e4inen M.: An experimental comparison of autoregressive and fourier-based descriptors in 2d shape classification. IEEE Trans. Pattern Anal. Mach. Intell. 17, 201\u2013207 (1995). doi: 10.1109\/34.368168","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"83_CR18","doi-asserted-by":"crossref","unstructured":"Lambert, G., Gao, H.: Discrimination properties of invariants using the line moments of vectorized contours. In: Proceedings of the 13th International Conference on Pattern Recognition, ICPR96, pp. 735\u2013739 (1996). doi: 10.1109\/ICPR.1996.546920","DOI":"10.1109\/ICPR.1996.546920"},{"issue":"5","key":"83_CR19","first-page":"240","volume":"9","author":"W. Liu","year":"1997","unstructured":"Liu W., Dori D.: A protocol for performance evaluation of line detection algorithms. Mach. Vis. Appl. 9(5), 240\u2013250 (1997). doi: 10.1007\/s001380050045","journal-title":"Mach. Vis. Appl."},{"issue":"4","key":"83_CR20","doi-asserted-by":"crossref","first-page":"424","DOI":"10.1109\/34.677280","volume":"20","author":"W. Liu","year":"1998","unstructured":"Liu W., Dori D.: Incremental arc segmentation algorithm and its evaluation. IEEE Trans. Pattern Anal.Mach. Intell. 20(4), 424\u2013431 (1998). doi: 10.1109\/34.677280","journal-title":"IEEE Trans. Pattern Anal.Mach. Intell."},{"key":"83_CR21","doi-asserted-by":"crossref","unstructured":"Lopresti, D., Nagy, G.: Issues in ground-truthing graphic documents. In: Graphics Recognition Algorithms and Applications. Lecture Notes on Computer Science, vol. 2390, pp. 46\u201366. Springer, Berlin (2001). doi: 10.1007\/3-540-45868-9","DOI":"10.1007\/3-540-45868-9"},{"key":"83_CR22","doi-asserted-by":"crossref","unstructured":"Lu, C., Shukla, M., Subramanya, S., Wu, Y.: Performance evaluation of desktop search engines. In: Proceedings of the IEEE International Conference on Information Reuse and Integration, IRI07, pp. 110\u2013115 (2007). doi: 10.1109\/IRI.2007.4296606","DOI":"10.1109\/IRI.2007.4296606"},{"key":"83_CR23","doi-asserted-by":"crossref","unstructured":"Lucas, S.: ICDAR 2005 text locating competition results. In: Proceedings of the Eighth International Conference on Document Analysis and Recognition, ICDAR05, pp. 80\u201384 (2005). doi: 10.1109\/ICDAR.2005.231","DOI":"10.1109\/ICDAR.2005.231"},{"issue":"2","key":"83_CR24","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1007\/s10032-004-0134-3","volume":"7","author":"S. Lucas","year":"2005","unstructured":"Lucas S., Panaretos A., Sosa L., Tang A., Wong S., Young R., Ashida K., Nagai H., Okamoto M., Yamamoto H., Miyao H., Zhu J., Ou W., Wolf C., Jolion J., Todoran L., Worring M., Lin X.: ICDAR 2003 robust reading competitions: entries, results, and future directions. Int. J. Doc. Anal. Recognit. 7(2), 105\u2013122 (2005). doi: 10.1007\/s10032-004-0134-3","journal-title":"Int. J. Doc. Anal. Recognit."},{"key":"83_CR25","unstructured":"Makhoul, J., Kubala, F., Schwartz, R., Weischedel, R.: Performance measures for information extraction. In: Proceedings of DARPA Broadcast News Workshop (1999)"},{"key":"83_CR26","doi-asserted-by":"crossref","unstructured":"Marcus, J.: A novel algorithm for HMM word spotting performance evaluation and error analysis. In: Proceedings of the IEEE International Conference on Acoustics, Speech, and Signal Processing, ICASSP92, pp. 89\u201392 (1992). doi: 10.1109\/ICASSP.1992.226113","DOI":"10.1109\/ICASSP.1992.226113"},{"issue":"5","key":"83_CR27","doi-asserted-by":"crossref","first-page":"593","DOI":"10.1016\/S0167-8655(00)00118-5","volume":"22","author":"H. M\u00fcller","year":"2001","unstructured":"M\u00fcller H., M\u00fcller W., Squire D., Marchand-Maillet S., Pun T.: Performance evaluation in content-based image retrieval: overview and proposals. Pattern Recognit. Lett. 22(5), 593\u2013601 (2001). doi: 10.1016\/S0167-8655(00)00118-5","journal-title":"Pattern Recognit. Lett."},{"key":"83_CR28","unstructured":"Neumann, T., Bender, M., Michel, S., Weikum, G.: A reproducible benchmark for P2P retrieval. In: Proceedings of the First International Workshop on Performance and Evaluation of Data Management Systems, ExpDB06, pp. 1\u20138 (2006)"},{"issue":"9","key":"83_CR29","doi-asserted-by":"crossref","first-page":"849","DOI":"10.1109\/34.790427","volume":"21","author":"I. Phillips","year":"1999","unstructured":"Phillips I., Chhabra A.: Empirical performance evaluation of graphics recognition systems. IEEE Trans. Pattern Anal. Mach. Intell. 21(9), 849\u2013870 (1999). doi: 10.1109\/34.790427","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"83_CR30","doi-asserted-by":"crossref","unstructured":"Rath, T., Manmatha, R.: Features for word spotting in historical manuscripts. In: Proceedings of the Seventh International Conf. on Document Analysis and Recognition, ICDAR03, pp. 218\u2013222 (2003). doi: 10.1109\/ICDAR.2003.1227662","DOI":"10.1109\/ICDAR.2003.1227662"},{"key":"83_CR31","volume-title":"Information Retrieval","author":"C. Rijsbergen van","year":"1979","unstructured":"van Rijsbergen C.: Information Retrieval. Butterworth-Heinemann Newton, MA, USA (1979)"},{"key":"83_CR32","doi-asserted-by":"crossref","unstructured":"Rusi\u00f1ol, M., Llad\u00f3s, J.: Word and symbol spotting using spatial organization of local descriptors. In: The Eighth IAPR International Workshop on Document Analysis Systems, DAS08, pp. 489\u2013496 (2008). doi: 10.1109\/DAS.2008.24","DOI":"10.1109\/DAS.2008.24"},{"key":"83_CR33","doi-asserted-by":"crossref","unstructured":"Smeaton, A., Over, P., Kraaij, W.: Evaluation campaigns and TRECVid. In: Proceedings of the 8th ACM International Workshop on Multimedia Information Retrieval, pp. 321\u2013330 (2006). doi: 10.1145\/1178677.1178722","DOI":"10.1145\/1178677.1178722"},{"issue":"8","key":"83_CR34","doi-asserted-by":"crossref","first-page":"1048","DOI":"10.1109\/TPAMI.2002.1023802","volume":"24","author":"J. Song","year":"2002","unstructured":"Song J., Su F., Tai C., Cai S.: An object-oriented progressive-simplification-based vectorization system for engineering drawings: Model, algorithm, and performance. IEEE Trans. Pattern Anal. Mach. Intell. 24(8), 1048\u20131060 (2002). doi: 10.1109\/TPAMI.2002.1023802","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"83_CR35","volume-title":"Fractals, Random Shapes and Point Fields (Methods of Geometrical Statistics)","author":"D. Stoyan","year":"1994","unstructured":"Stoyan D., Stoyan H.: Fractals, Random Shapes and Point Fields (Methods of Geometrical Statistics). John Wiley & Sons, Chichester (1994)"},{"key":"83_CR36","doi-asserted-by":"crossref","unstructured":"Tabbone, S., Zuwala, D.: An indexing method for graphical documents. In: Proceedings of the Ninth International Conference on Document Analysis and Recognition, ICDAR07, pp. 789\u2013793 (2007). doi: 10.1109\/ICDAR.2007.4377023","DOI":"10.1109\/ICDAR.2007.4377023"},{"key":"83_CR37","doi-asserted-by":"crossref","unstructured":"Tombre, K., Lamiroy, B.: Graphics recognition\u2014from re-engineering to retrieval. In: Proceedings of the Seventh International Conference on Document Analysis and Recognition, ICDAR03, pp. 148\u2013155 (2003)","DOI":"10.1109\/ICDAR.2003.1227650"},{"key":"83_CR38","doi-asserted-by":"crossref","unstructured":"Valveny, E., Dosch, P.: Symbol recognition contest: a synthesis. In: Graphics Recognition, Recent Advances and Perspectives. Lecture Notes on Computer Science, vol. 3088, pp. 368\u2013385. Springer, Berlin (2004). doi: 10.1007\/b99011","DOI":"10.1007\/b99011"},{"key":"83_CR39","doi-asserted-by":"crossref","unstructured":"Valveny, E., Dosch, P.: Report on the second symbol recognition contest. In: Graphics Recognition. Ten Years Review and Future Perspectives. Lecture Notes on Computer Science, vol. 3926, pp. 381\u2013397. Springer, Berlin (2006). doi: 10.1007\/11767978","DOI":"10.1007\/11767978"},{"issue":"1","key":"83_CR40","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1007\/s10032-006-0033-x","volume":"9","author":"E. Valveny","year":"2007","unstructured":"Valveny E., Dosch P., Winstanley A., Zhou Y., Yang S., Yan L., Liu W., Elliman D., Delalandre M., Trupin E., Adam S., Ogier J.: A general framework for the evaluation of symbol recognition methods. Int. J. Doc. Anal. Recognit. 9(1), 59\u201374 (2007). doi: 10.1007\/s10032-006-0033-x","journal-title":"Int. J. Doc. Anal. Recognit."},{"issue":"4","key":"83_CR41","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1007\/s10032-006-0014-0","volume":"8","author":"C. Wolf","year":"2006","unstructured":"Wolf C., Jolion J.: Object count\/area graphs for the evaluation of object detection and segmentation algorithms. Int. J. Doc. Anal. Recognit. 8(4), 280\u2013296 (2006). doi: 10.1007\/s10032-006-0014-0","journal-title":"Int. J. Doc. Anal. Recognit."}],"container-title":["International Journal on Document Analysis and Recognition (IJDAR)"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10032-009-0083-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10032-009-0083-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10032-009-0083-y","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,29]],"date-time":"2019-05-29T07:34:32Z","timestamp":1559115272000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10032-009-0083-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009,3,19]]},"references-count":41,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2009,7]]}},"alternative-id":["83"],"URL":"https:\/\/doi.org\/10.1007\/s10032-009-0083-y","relation":{},"ISSN":["1433-2833","1433-2825"],"issn-type":[{"value":"1433-2833","type":"print"},{"value":"1433-2825","type":"electronic"}],"subject":[],"published":{"date-parts":[[2009,3,19]]}}}