{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,4]],"date-time":"2026-06-04T05:43:33Z","timestamp":1780551813619,"version":"3.54.1"},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2008,12,1]],"date-time":"2008-12-01T00:00:00Z","timestamp":1228089600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Lang Resources & Evaluation"],"published-print":{"date-parts":[[2008,12]]},"DOI":"10.1007\/s10579-008-9079-3","type":"journal-article","created":{"date-parts":[[2008,12,4]],"date-time":"2008-12-04T12:20:26Z","timestamp":1228393226000},"page":"361-393","source":"Crossref","is-referenced-by-count":15,"title":["Evaluation of machine learning-based information extraction algorithms: criticisms and recommendations"],"prefix":"10.1007","volume":"42","author":[{"given":"Alberto","family":"Lavelli","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mary Elaine","family":"Califf","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fabio","family":"Ciravegna","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dayne","family":"Freitag","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Claudio","family":"Giuliano","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nicholas","family":"Kushmerick","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lorenza","family":"Romano","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Neil","family":"Ireson","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2008,12,5]]},"reference":[{"key":"9079_CR2","unstructured":"Califf, M. E. (1998). Relational learning techniques for natural language information extraction. Ph.D. thesis, University of Texas at Austin."},{"key":"9079_CR1","first-page":"177","volume":"4","author":"M. Califf","year":"2003","unstructured":"Califf, M., & Mooney, R. (2003). Bottom-up relational learning of pattern matching rules for information extraction. Journal of Machine Learning Research, 4, 177\u2013210.","journal-title":"Journal of Machine Learning Research"},{"key":"9079_CR3","unstructured":"Chieu, H. L., & Ng, H. T. (2002). Probabilistic reasoning for entity and relation recognition. In Proceedings of the 19th National Conference on Artificial Intelligence (AAAI 2002)."},{"issue":"3","key":"9079_CR4","first-page":"409","volume":"19","author":"N. Chinchor","year":"1993","unstructured":"Chinchor, N., Hirschman, L., & Lewis, D. D. (1993). Evaluating message understanding systems: An analysis of the third Message Understanding Conference (MUC-3). Computational Linguistics, 19(3), 409\u2013449.","journal-title":"Computational Linguistics"},{"key":"9079_CR5","unstructured":"Ciravegna, F. (2001a). Adaptive information extraction from text by rule induction and generalisation. In Proceedings of 17th International Joint Conference on Artificial Intelligence (IJCAI-01). Seattle, WA."},{"key":"9079_CR6","unstructured":"Ciravegna, F. (2001b). (LP)2, an adaptive algorithm for information extraction from web-related texts. In Proceedings of the IJCAI-2001 Workshop on Adaptive Text Extraction and Mining. Seattle, WA."},{"key":"9079_CR7","doi-asserted-by":"crossref","unstructured":"Ciravegna, F., Dingli, A., Petrelli, D., & Wilks, Y. (2002). User-system cooperation in document annotation based on information extraction. In Proceedings of the 13th International Conference on Knowledge Engineering and Knowledge Management (EKAW02).","DOI":"10.1007\/3-540-45810-7_15"},{"issue":"2","key":"9079_CR8","doi-asserted-by":"crossref","first-page":"145","DOI":"10.1017\/S135132490400333X","volume":"10","author":"F. Ciravegna","year":"2004","unstructured":"Ciravegna, F., & Lavelli, A. (2004). LearningPinocchio: Adaptive information extraction for real world applications. Journal of Natural Language Engineering, 10(2), 145\u2013165.","journal-title":"Journal of Natural Language Engineering"},{"key":"9079_CR9","unstructured":"Daelemans, W., & Hoste, V. (2002). Evaluation of machine learning methods for natural language processing tasks. In Proceedings of the Third International Conference on Language Resources and Evaluation (LREC 2002). Las Palmas, Spain."},{"key":"9079_CR10","doi-asserted-by":"crossref","unstructured":"Daelemans, W., Hoste, V., Meulder, F. D., & Naudts, B. (2003). Combined optimization of feature selection and algorithm parameters in machine learning of language. In Proceedings of the 14th European Conference on Machine Learning (ECML 2003). Cavtat-Dubronik, Croatia.","DOI":"10.1007\/978-3-540-39857-8_10"},{"key":"9079_CR11","unstructured":"De Sitter, A., & Daelemans, W. (2003). Information extraction via double classification. In Proceedings of the ECML\/PKDD 2003 Workshop on Adaptive Text Extraction and Mining (ATEM 2003). Cavtat-Dubronik, Croatia."},{"key":"9079_CR12","unstructured":"Douthat, A. (1998). The Message Understanding Conference scoring software user\u2019s manual. In Proceedings of the 7th Message Understanding Conference (MUC-7). http:\/\/www.itl.nist.gov\/iaui\/894.02\/related_projects\/muc\/muc_sw\/muc_sw_manual.html ."},{"key":"9079_CR13","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4899-4541-9","volume-title":"An introduction to the bootstrap","author":"B. Efron","year":"1993","unstructured":"Efron, B., & Tibshirani, R. J. (1993). An introduction to the bootstrap. New York: Chapman and Hall."},{"key":"9079_CR14","doi-asserted-by":"crossref","unstructured":"Finkel, J. R., Grenager, T., & Manning, C. (2005). Incorporating non-local information into information extraction systems by Gibbs sampling. In Proceedings of the 43rd Annual Meeting on Association for Computational Linguistics (ACL 2005).","DOI":"10.3115\/1219840.1219885"},{"key":"9079_CR15","unstructured":"Finn, A., & Kushmerick, N. (2004a). Information extraction by convergent boundary classification. In Proceedings of the AAAI 2004 Workshop on Adaptive Text Extraction and Mining (ATEM 2004). San Jose, California."},{"key":"9079_CR16","doi-asserted-by":"crossref","unstructured":"Finn, A., & Kushmerick, N. (2004b). Multi-level boundary classification for information extraction. In Proceedings of the 15th European Conference on Machine Learning. Pisa, Italy.","DOI":"10.1007\/978-3-540-30115-8_13"},{"key":"9079_CR17","unstructured":"Freitag, D. (1997). Using grammatical inference to improve precision in information extraction. In Proceedings of the ICML-97 Workshop on Automata Induction, Grammatical Inference, and Language Acquisition. Nashville, Tennessee."},{"key":"9079_CR18","unstructured":"Freitag, D. (1998). Machine learning for information extraction in informal domains. Ph.D. thesis, Carnegie Mellon University."},{"key":"9079_CR19","unstructured":"Freitag, D., & Kushmerick, N. (2000). Boosted wrapper induction. In Proceedings of the 17th National Conference on Artificial Intelligence (AAAI 2000). Austin, Texas."},{"key":"9079_CR20","unstructured":"Habert, B., Adda, G., Adda-Decker, M., de Mareuil, P. B., Ferrari, S., Ferret, O., Illouz, G., & Paroubek, P. (1998). Towards tokenization evaluation. In Proceedings of 1st International Conference on Language Resources and Evaluation (LREC-98). Granada, Spain."},{"issue":"4","key":"9079_CR21","doi-asserted-by":"crossref","first-page":"281","DOI":"10.1006\/csla.1998.0102","volume":"12","author":"L. Hirschman","year":"1998","unstructured":"Hirschman, L. (1998). The evolution of evaluation: Lessons from the Message Understanding Conferences. Computer Speech and Language, 12(4), 281\u2013305.","journal-title":"Computer Speech and Language"},{"issue":"4","key":"9079_CR22","doi-asserted-by":"crossref","first-page":"311","DOI":"10.1017\/S1351324902003005","volume":"8","author":"V. Hoste","year":"2002","unstructured":"Hoste, V., Hendrickx, I., Daelemans, W., & van den Bosch, A. (2002). Parameter optimization for machine-learning of word sense disambiguation. Natural Language Engineering, 8(4), 311\u2013325.","journal-title":"Natural Language Engineering"},{"key":"9079_CR23","doi-asserted-by":"crossref","unstructured":"Ireson, N., Ciravegna, F., Califf, M. E., Freitag, D., Kushmerick, N., & Lavelli, A. (2005). Evaluating machine learning for information extraction. In Proceedings of 22nd International Conference on Machine Learning (ICML 2005). Bonn, Germany.","DOI":"10.1145\/1102351.1102395"},{"key":"9079_CR24","unstructured":"Iria, J., & Ciravegna, F. (2006). A methodology and tool for representing language resources for information extraction. In Proceedings of the 5th International Conference on Language Resources and Evaluation (LREC 2006). Genoa, Italy."},{"key":"9079_CR25","unstructured":"Kosala, R., & Blockeel, H. (2000). Instance-based wrapper induction. In Proceedings of the Tenth Belgian-Dutch Conference on Machine Learning (Benelearn 2000). Tilburg, The Netherlands, pp. 61\u201368."},{"issue":"1\u20132","key":"9079_CR26","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1016\/S0004-3702(99)00100-9","volume":"118","author":"N. Kushmerick","year":"2000","unstructured":"Kushmerick, N. (2000). Wrapper induction: Efficency and expressiveness. Artificial Intelligence, 118(1\u20132), 15\u201368.","journal-title":"Artificial Intelligence"},{"key":"9079_CR27","doi-asserted-by":"crossref","unstructured":"Li, Y., Bontcheva, K., & Cunningham, H. (2005a). SVM based learning system for information extraction. In J. Winkler, M. Niranjan, & N. Lawrence (Eds.), Deterministic and statistical methods in machine learning, Vol. 3635 of LNAI. (pp. 319\u2013339). Springer Verlag.","DOI":"10.1007\/11559887_19"},{"key":"9079_CR28","doi-asserted-by":"crossref","unstructured":"Li, Y., Bontcheva, K., & Cunningham, H. (2005b). Using uneven margins SVM and perceptron for information extraction. In Proceedings of the Ninth Conference on Computational Natural Language Learning (CONLL 2005).","DOI":"10.3115\/1706543.1706556"},{"key":"9079_CR29","unstructured":"Makhoul, J., Kubala, F., Schwartz, R., & Weischedel, R. (1999), Performance measures for information extraction. In Proceedings of the DARPA Broadcast News Workshop. http:\/\/www.nist.gov\/speech\/publications\/darpa99\/pdf\/dir10.pdf ."},{"key":"9079_CR30","volume-title":"Computer intensive methods for testing hypotheses: An introduction","author":"E. W. Noreen","year":"1989","unstructured":"Noreen, E. W. (1989). Computer Intensive Methods for Testing Hypotheses: An Introduction. New York: Wiley."},{"key":"9079_CR31","unstructured":"Peshkin, L., & Pfeffer, A. (2003). Bayesian information extraction network. In Proceedings of 18th International Joint Conference on Artificial Intelligence (IJCAI 2003). Acapulco, Mexico."},{"key":"9079_CR32","unstructured":"RISE. (1998). A repository of online information sources used in information extraction tasks. [ http:\/\/www.isi.edu\/info-agents\/RISE\/index.html ] Information Sciences Institute\/USC."},{"key":"9079_CR33","unstructured":"Roth, D., & Yih, W. (2001). Relational learning via propositional algorithms: An information extraction case study. In Proceedings of 17th International Joint Conference on Artificial Intelligence (IJCAI-01). Seattle, WA."},{"key":"9079_CR34","unstructured":"Roth, D., & Yih, W. (2002). Relational learning via propositional algorithms: An information extraction case study. Technical Report UIUCDCS-R-2002-2300, Department of Computer Science, University of Illinois at Urbana-Champaign."},{"key":"9079_CR35","first-page":"1751","volume":"6","author":"G. Sigletos","year":"2005","unstructured":"Sigletos, G., Paliouros, G., Spyropoulos, C., & Hatzopoulos, M. (2005). Combining information extraction systems using voting and stacked generalization. Journal of Machine Learning Research, 6, 1751\u20131782.","journal-title":"Journal of Machine Learning Research"},{"issue":"1\u20133","key":"9079_CR36","doi-asserted-by":"crossref","first-page":"233","DOI":"10.1023\/A:1007562322031","volume":"34","author":"S. Soderland","year":"1999","unstructured":"Soderland, S. (1999). Learning information extraction rules for semi-structured and free text. Machine Learning, 34(1\u20133), 233\u2013272.","journal-title":"Machine Learning"},{"key":"9079_CR37","unstructured":"Sutton, C., & McCallum, A. (2004). Collective segmentation and labeling of distant entities. In Proceedings of the ICML Workshop on Statistical Relational Learning and Its Connections to Other Fields."}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-008-9079-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10579-008-9079-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-008-9079-3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,30]],"date-time":"2019-05-30T18:21:15Z","timestamp":1559240475000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10579-008-9079-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2008,12]]},"references-count":37,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2008,12]]}},"alternative-id":["9079"],"URL":"https:\/\/doi.org\/10.1007\/s10579-008-9079-3","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"value":"1574-020X","type":"print"},{"value":"1574-0218","type":"electronic"}],"subject":[],"published":{"date-parts":[[2008,12]]}}}