{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,30]],"date-time":"2025-10-30T07:11:15Z","timestamp":1761808275269,"version":"3.37.3"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2019,4,1]],"date-time":"2019-04-01T00:00:00Z","timestamp":1554076800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2019,4,1]],"date-time":"2019-04-01T00:00:00Z","timestamp":1554076800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100007136","name":"Secretar\u00eda de Estado de Investigaci\u00f3n, Desarrollo e Innovaci\u00f3n","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100007136","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100010198","name":"Ministerio de Econom\u00eda, Industria y Competitividad, Gobierno de Espa\u00f1a","doi-asserted-by":"publisher","award":["TIN2015-71785-R"],"award-info":[{"award-number":["TIN2015-71785-R"]}],"id":[{"id":"10.13039\/501100010198","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2019,12]]},"DOI":"10.1007\/s10791-019-09355-y","type":"journal-article","created":{"date-parts":[[2019,4,1]],"date-time":"2019-04-01T14:03:28Z","timestamp":1554127408000},"page":"581-619","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["A comparison of filtering evaluation metrics based on formal constraints"],"prefix":"10.1007","volume":"22","author":[{"given":"Enrique","family":"Amig\u00f3","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5341-9337","authenticated-orcid":false,"given":"Julio","family":"Gonzalo","sequence":"additional","affiliation":[]},{"given":"Felisa","family":"Verdejo","sequence":"additional","affiliation":[]},{"given":"Damiano","family":"Spina","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,4,1]]},"reference":[{"key":"9355_CR1","doi-asserted-by":"crossref","unstructured":"Agresti, A., & Hitchcock, D.\u00a0B. (2005). Bayesian inference for categorical data analysis: A survey. Technical report.","DOI":"10.1007\/s10260-005-0121-y"},{"key":"9355_CR2","unstructured":"Amig\u00f3, E., Artiles, J., Gonzalo, J., Spina, D., Liu, B., & Corujo, A. (2010). WePS3 evaluation campaign: Overview of the on-line reputation management task. In 2nd Web people search evaluation workshop (WePS 2010), CLEF 2010 conference, Padova Italy."},{"key":"9355_CR3","doi-asserted-by":"crossref","unstructured":"Amig\u00f3, E., Corujo, A., Gonzalo, J., Meij, E., & de\u00a0Rijke, M. (2012). Overview of RepLab 2012: Evaluating online reputation management systems. In CLEF (online working notes\/labs\/workshop).","DOI":"10.1007\/978-3-642-40802-1_31"},{"key":"9355_CR4","doi-asserted-by":"crossref","unstructured":"Amigo, E., Fang, H., Mizzaro, S., & Zhai, C. (2017). Axiomatic thinking for information retrieval and related tasks. In Proceedings of the 40th international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201917, pp. 1419\u20131420, New York, 2017. ACM.","DOI":"10.1145\/3077136.3084369"},{"issue":"4","key":"9355_CR5","doi-asserted-by":"publisher","first-page":"461","DOI":"10.1007\/s10791-008-9066-8","volume":"12","author":"E Amig\u00f3","year":"2009","unstructured":"Amig\u00f3, E., Gonzalo, J., Artiles, J., & Verdejo, F. (2009). A comparison of extrinsic clustering evaluation metrics based on formal constraints. Information Retrieval, 12(4), 461\u2013486.","journal-title":"Information Retrieval"},{"key":"9355_CR6","doi-asserted-by":"crossref","unstructured":"Amig\u00f3, E., Gonzalo, J, & Verdejo, F. (2013). A generic measure for document organization tasks. In Proceedings of ACM SIGIR, pp. 643\u2013652. ACM Press.","DOI":"10.1145\/2484028.2484081"},{"key":"9355_CR7","doi-asserted-by":"crossref","unstructured":"Amig\u00f3, E., Spina, D., & Carrillo-de-Albornoz, J. (2018). An axiomatic analysis of diversity evaluation metrics: Introducing the rank-biased utility metric. In CoRR, abs\/1805.02334.","DOI":"10.1145\/3209978.3210024"},{"key":"9355_CR8","unstructured":"Androutsopoulos, I., Koutsias, J., Chandrinos, K., Paliouras, G., & Spyropoulos, C.\u00a0D. (2000). An evaluation of naive bayesian anti-spam filtering. In CoRR, cs.CL\/0006013."},{"key":"9355_CR9","doi-asserted-by":"publisher","first-page":"1145","DOI":"10.1016\/S0031-3203(96)00142-2","volume":"30","author":"Andrew P Bradley","year":"1997","unstructured":"Bradley, Andrew P. (1997). The use of the area under the ROC curve in the evaluation of machine learning algorithms. Pattern Recognition, 30, 1145\u20131159.","journal-title":"Pattern Recognition"},{"key":"9355_CR10","doi-asserted-by":"crossref","unstructured":"Busin, L., & Mizzaro, S. (2013). Axiometrics: An axiomatic approach to information retrieval effectiveness metrics. In Proceedings of the 2013 conference on the theory of information retrieval, ICTIR \u201913, pp. 8:22\u20138:29, New York, NY, 2013. ACM.","DOI":"10.1145\/2499178.2499182"},{"key":"9355_CR11","doi-asserted-by":"crossref","unstructured":"Callan, J. (1996). Document filtering with inference networks. In Proceedings of the nineteenth annual international ACM SIGIR conference on research and development in information retrieval, pp. 262\u2013269.","DOI":"10.1145\/243199.243273"},{"key":"9355_CR12","doi-asserted-by":"crossref","unstructured":"Caruana, R., & Niculescu-Mizil, A. (2005). An empirical comparison of supervised learning algorithms using different performance metrics. In Proceedings of 23rd international conference machine learning (ICML06), pp. 161\u2013168.","DOI":"10.1145\/1143844.1143865"},{"key":"9355_CR13","doi-asserted-by":"crossref","unstructured":"Clinchant, S., & Gaussier, E. (2011). Is document frequency important for PRF? In Advances in information retrieval theory, pp. 89\u2013100. Springer.","DOI":"10.1007\/978-3-642-23318-0_10"},{"issue":"1","key":"9355_CR14","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J. (1960). A coefficient of agreement for nominal scales. Educational and Psychological Measurement, 20(1), 37.","journal-title":"Educational and Psychological Measurement"},{"key":"9355_CR15","doi-asserted-by":"crossref","unstructured":"Cormack, G., & Lynam, T. (2005). TREC 2005 spam track overview. In Proceedings of the fourteenth text retrieval conference 8TREC 2005).","DOI":"10.6028\/NIST.SP.500-266.spam-overview"},{"key":"9355_CR16","unstructured":"Fang, H. (2008). A re-examination of query expansion using lexical resources. In ACL, vol. 2008, pp. 139\u2013147. Citeseer."},{"key":"9355_CR17","doi-asserted-by":"crossref","unstructured":"Fang, H., Tao, T., & Zhai, C. X. (2004). A formal study of information retrieval heuristics. In Proceedings of the 27th annual international ACM SIGIR conference on Research and development in information retrieval, pp. 49\u201356. ACM.","DOI":"10.1145\/1008992.1009004"},{"key":"9355_CR18","doi-asserted-by":"crossref","unstructured":"Fang, H., & Zhai, C. X. (2006). Semantic term matching in axiomatic approaches to information retrieval. In Proceedings of the 29th annual international ACM SIGIR conference on research and development in information retrieval, pp. 115\u2013122. ACM.","DOI":"10.1145\/1148170.1148193"},{"key":"9355_CR19","doi-asserted-by":"crossref","unstructured":"Fang, H, & Zhai, C. X. (2014). Axiomatic analysis and optimization of information retrieval models. In Proceedings of the 37th international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201914, pp. 1288\u20131288, New York, NY, 2014. ACM.","DOI":"10.1145\/2600428.2602294"},{"key":"9355_CR20","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1007\/s10994-007-5011-0","volume":"68","author":"T Fawcett","year":"2007","unstructured":"Fawcett, T., & Niculescu-Mizil, A. (2007). PAV and the ROC convex hull. Machine Learning, 68, 97\u2013106.","journal-title":"Machine Learning"},{"issue":"1","key":"9355_CR21","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1016\/j.patrec.2008.08.010","volume":"30","author":"C Ferri","year":"2009","unstructured":"Ferri, C., Hern\u00e1ndez-Orallo, J., & Modroiu, R. (2009). An experimental comparison of performance measures for classification. Pattern Recognition Letters, 30(1), 27\u201338.","journal-title":"Pattern Recognition Letters"},{"key":"9355_CR22","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1111\/j.2517-6161.1952.tb00104.x","volume":"14","author":"IJ Good","year":"1952","unstructured":"Good, I. J. (1952). Rational decisions. Journal of the Royal Statistical Society Series B (Methodological), 14, 107\u2013114.","journal-title":"Journal of the Royal Statistical Society Series B (Methodological)"},{"key":"9355_CR23","doi-asserted-by":"crossref","unstructured":"Hedin, B., Tomlinson, S., Baron, J.\u00a0R., & Oard, D.\u00a0W. (2009). Overview of the TREC 2009 legal track.","DOI":"10.6028\/NIST.SP.500-278.legal-overview"},{"key":"9355_CR24","doi-asserted-by":"crossref","unstructured":"Hoashi, K., Matsumoto, K., Inoue, N., & Hashimoto, K. (2000). Document filtering method using non-relevant information profile. In Proceedings of the 23rd annual international ACM SIGIR conference on research and development in information retrieval, SIGIR \u201900, pp. 176\u2013183, New York, NY, 2000. ACM.","DOI":"10.1145\/345508.345573"},{"key":"9355_CR25","first-page":"33","volume":"6","author":"David A Hull","year":"1997","unstructured":"Hull, David A. (1997). The TREC-6 filtering track: Description and analysis. Proceedings of the TREC, 6, 33\u201356.","journal-title":"Proceedings of the TREC"},{"key":"9355_CR26","unstructured":"Hull, D.\u00a0A. (1998). The TREC-7 filtering track: Description and analysis. In E.\u00a0M. Voorhees and D.\u00a0K. Harman, editors, Proceedings of TREC-7, 7th text retrieval conference, pp. 33\u201356, Gaithersburg, US, 1998. National Institute of Standards and Technology, Gaithersburg, US."},{"key":"9355_CR27","doi-asserted-by":"crossref","unstructured":"Karimzadehgan, M., & Zhai, C. X. (2012). Axiomatic analysis of translation language model for information retrieval. In Advances in information retrieval, pp. 268\u2013280. Springer, Berlin.","DOI":"10.1007\/978-3-642-28997-2_23"},{"issue":"2","key":"9355_CR28","first-page":"1","volume":"23","author":"BP Karon","year":"1958","unstructured":"Karon, B. P., & Alexander, I. E. (1958). Association and estimation in contingency tables. Journal of the American Statistical Association, 23(2), 1\u201328.","journal-title":"Journal of the American Statistical Association"},{"key":"9355_CR29","doi-asserted-by":"crossref","unstructured":"Krishnamurthy, B., Gill, P., & Arlitt, M. (2008). A few chirps about twitter. In WOSP \u201908: Proceedings of the first workshop on online social networks, pp. 19\u201324, New York, NY, 2008. ACM.","DOI":"10.1145\/1397735.1397741"},{"key":"9355_CR30","unstructured":"Le, A., Ajot, J., Przybocki, M., & Strassel, S. (2010). Document image collection using Amazon\u2019s mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s mechanical turk, pp. 45\u201352, Los Angeles, June 2010. Association for Computational Linguistics."},{"key":"9355_CR31","unstructured":"Ling, C.\u00a0X., Huang, J., & Zhang, H. (2003). AUC: A statistically consistent and more discriminating measure than accuracy. In IJCAI, pp. 519\u2013526."},{"key":"9355_CR32","doi-asserted-by":"crossref","unstructured":"Lv, Y., & Zhai, C. X. (2011). Lower-bounding term frequency normalization. In Proceedings of the 20th ACM international conference on information and knowledge management, CIKM \u201911, pp. 7\u201316, New York, NY, 2011. ACM.","DOI":"10.1145\/2063576.2063584"},{"key":"9355_CR33","volume-title":"Machine learning","author":"TM Mitchell","year":"1997","unstructured":"Mitchell, T. M. (1997). Machine learning. New York: McGraw Hill."},{"key":"9355_CR34","doi-asserted-by":"publisher","first-page":"339","DOI":"10.1007\/978-1-4471-2099-5_35","volume-title":"SIGIR \u201994","author":"Michael Persin","year":"1994","unstructured":"Persin, Michael. (1994). Document filtering for fast ranking. In Proceedings of the 17th annual international ACM SIGIR conference on Research and development in information retrieval, SIGIR \u201994, pp. 339\u2013348, New York, NY, 1994. Springer, New York."},{"key":"9355_CR35","unstructured":"Provost, F.\u00a0J., & Fawcett, T. (1997). Analysis and visualization of classifier performance: Comparison under imprecise class and cost distributions. In Knowledge discovery and data mining, pp. 43\u201348."},{"key":"9355_CR36","doi-asserted-by":"crossref","unstructured":"Qi, Haoliang, Yang, Muyun, He, Xiaoning, & Li, Sheng. (2010). Re-examination on lam% in spam filtering. In Proceedings of the SIGIR 2010 conference, Geneva, Switzerland.","DOI":"10.1145\/1835449.1835601"},{"key":"9355_CR37","unstructured":"Robertson, S., & Hull, D.\u00a0A. (2001). The TREC-9 filtering track final report. In Proceedings of TREC-9, pp. 25\u201340."},{"key":"9355_CR38","doi-asserted-by":"crossref","unstructured":"Schapire, R.\u00a0E., Singer, Y., & Singhal, A. (1998). Boosting and Rocchio applied to text filtering. In Proceedings of ACM SIGIR, pp. 215\u2013223. ACM Press.","DOI":"10.1145\/290941.290996"},{"key":"9355_CR39","doi-asserted-by":"crossref","unstructured":"Sebastiani, F. (2015). An axiomatically derived measure for the evaluation of classification algorithms. In ICTIR, pp. 11\u201320.","DOI":"10.1145\/2808194.2809449"},{"key":"9355_CR40","unstructured":"Sokolova, M. (2006). Assessing invariance properties of evaluation measures. In Proceedings of NIPS\u201906 workshop on testing deployable learning and decision systems."},{"key":"9355_CR41","doi-asserted-by":"crossref","unstructured":"Sokolova, M., Japkowicz, N., & Szpakowicz, S. (2006). Beyond accuracy, F-score and ROC: A family of discriminant measures for performance evaluation. AI 2006: Advances in artificial intelligence, pp. 1015\u20131021.","DOI":"10.1007\/11941439_114"},{"key":"9355_CR42","doi-asserted-by":"publisher","first-page":"327","DOI":"10.1037\/0033-295X.84.4.327","volume":"84","author":"A Tversky","year":"1977","unstructured":"Tversky, A. (1977). Features of similarity. Psychological Review, 84, 327\u2013352.","journal-title":"Psychological Review"},{"issue":"4","key":"9355_CR43","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1108\/eb026584","volume":"30","author":"C Van Rijsbergen","year":"1974","unstructured":"Van Rijsbergen, C. (1974). Foundation of evaluation. Journal of Documentation, 30(4), 365\u2013373.","journal-title":"Journal of Documentation"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-019-09355-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-019-09355-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-019-09355-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,16]],"date-time":"2024-07-16T13:19:54Z","timestamp":1721135994000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-019-09355-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,4,1]]},"references-count":43,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2019,12]]}},"alternative-id":["9355"],"URL":"https:\/\/doi.org\/10.1007\/s10791-019-09355-y","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"type":"print","value":"1386-4564"},{"type":"electronic","value":"1573-7659"}],"subject":[],"published":{"date-parts":[[2019,4,1]]},"assertion":[{"value":"14 April 2016","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 March 2019","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 April 2019","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}