{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T16:05:47Z","timestamp":1774454747051,"version":"3.50.1"},"reference-count":37,"publisher":"Public Library of Science (PLoS)","issue":"3","license":[{"start":{"date-parts":[[2018,3,13]],"date-time":"2018-03-13T00:00:00Z","timestamp":1520899200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.plosone.org"],"crossmark-restriction":false},"short-container-title":["PLoS ONE"],"DOI":"10.1371\/journal.pone.0194317","type":"journal-article","created":{"date-parts":[[2018,3,13]],"date-time":"2018-03-13T17:33:10Z","timestamp":1520962390000},"page":"e0194317","update-policy":"https:\/\/doi.org\/10.1371\/journal.pone.corrections_policy","source":"Crossref","is-referenced-by-count":24,"title":["How to evaluate sentiment classifiers for Twitter time-ordered data?"],"prefix":"10.1371","volume":"13","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5466-0608","authenticated-orcid":true,"given":"Igor","family":"Mozeti\u010d","sequence":"first","affiliation":[]},{"given":"Luis","family":"Torgo","sequence":"additional","affiliation":[]},{"given":"Vitor","family":"Cerqueira","sequence":"additional","affiliation":[]},{"given":"Jasmina","family":"Smailovi\u0107","sequence":"additional","affiliation":[]}],"member":"340","published-online":{"date-parts":[[2018,3,13]]},"reference":[{"issue":"1-2","key":"ref1","doi-asserted-by":"crossref","first-page":"117","DOI":"10.1016\/0377-0427(95)00011-9","article-title":"More effective time-series analysis and forecasting","volume":"64","author":"OD Anderson","year":"1995","journal-title":"Journal of Computational and Applied Mathematics"},{"key":"ref2","unstructured":"Agarwal A, Xie B, Vovsha I, Rambow O, Passonneau R. Sentiment analysis of Twitter data. In: Proc. Workshop on Languages in Social Media. ACL; 2011. p. 30\u201338."},{"key":"ref3","unstructured":"Mohammad SM, Kiritchenko S, Zhu X. NRC-Canada: Building the state-of-the-art in sentiment analysis of tweets. arXiv preprint arXiv:13086242; 2013."},{"key":"ref4","doi-asserted-by":"crossref","unstructured":"Bermingham A, Smeaton AF. Classifying sentiment in microblogs: is brevity an advantage? In: Proc. 19th ACM Intl. Conference on Information and Knowledge Management. ACM; 2010. p. 1833\u20131836.","DOI":"10.1145\/1871437.1871741"},{"key":"ref5","unstructured":"Saif H, Fern\u00e1ndez M, He Y, Alani H. Evaluation datasets for Twitter sentiment analysis: A survey and a new dataset, the STS-Gold. In: Proc. 1st Intl. Workshop on Emotion and Sentiment in Social and Expressive Media: Approaches and Perspectives from AI (ESSEM); 2013."},{"key":"ref6","doi-asserted-by":"crossref","unstructured":"Saif H, He Y, Alani H. Semantic sentiment analysis of Twitter. In: Proc. Intl. Semantic Web Conference (ISWC). Springer; 2012. p. 508\u2013524.","DOI":"10.1007\/978-3-642-35176-1_32"},{"key":"ref7","doi-asserted-by":"crossref","unstructured":"Wang X, Wei F, Liu X, Zhou M, Zhang M. Topic sentiment analysis in Twitter: a graph-based hashtag sentiment classification approach. In: Proc. 20th ACM Intl. Conference on Information and Knowledge Management. ACM; 2011. p. 1031\u20131040.","DOI":"10.1145\/2063576.2063726"},{"key":"ref8","doi-asserted-by":"crossref","unstructured":"Bifet A, Frank E. Sentiment knowledge discovery in Twitter streaming data. In: Proc. 13th Intl. Conference on Discovery Science; 2010. p. 1\u201315.","DOI":"10.1007\/978-3-642-16184-1_1"},{"key":"ref9","first-page":"215","article-title":"Proc. Advances in Intelligent Data Analysis XIII (IDA)","author":"N Moniz","year":"2014"},{"key":"ref10","doi-asserted-by":"crossref","first-page":"40","DOI":"10.1214\/09-SS054","article-title":"A survey of cross-validation procedures for model selection","volume":"4","author":"S Arlot","year":"2010","journal-title":"Statistics Surveys"},{"issue":"4","key":"ref11","doi-asserted-by":"crossref","first-page":"437","DOI":"10.1016\/S0169-2070(00)00065-0","article-title":"Out-of-sample tests of forecasting accuracy: an analysis and review","volume":"16","author":"LJ Tashman","year":"2000","journal-title":"International Journal of Forecasting"},{"key":"ref12","doi-asserted-by":"crossref","first-page":"192","DOI":"10.1016\/j.ins.2011.12.028","article-title":"On the use of cross-validation for time series predictor evaluation","volume":"191","author":"C Bergmeir","year":"2012","journal-title":"Information Sciences"},{"issue":"9","key":"ref13","doi-asserted-by":"crossref","first-page":"1056","DOI":"10.1287\/mnsc.35.9.1056","article-title":"Evaluation of aggregate and individual forecast method selection rules","volume":"35","author":"R Fildes","year":"1989","journal-title":"Management Science"},{"key":"ref14","unstructured":"Torgo L. An infra-structure for performance estimation and experimental comparison of predictive models in R. arXiv preprint arXiv:14120436; 2014."},{"key":"ref15","article-title":"Data stream mining: a practical approach","author":"A Bifet","year":"2009"},{"issue":"1","key":"ref16","doi-asserted-by":"crossref","first-page":"128","DOI":"10.1007\/s10618-010-0201-y","article-title":"Learning model trees from evolving data streams","volume":"23","author":"E Ikonomovska","year":"2011","journal-title":"Data Mining and Knowledge Discovery"},{"key":"ref17","doi-asserted-by":"crossref","unstructured":"Snijders TAB. On cross-validation for predictor evaluation in time series. In: Proc. Workshop On Model Uncertainty and its Statistical Implications. Springer; 1988. p. 56\u201369.","DOI":"10.1007\/978-3-642-61564-1_4"},{"key":"ref18","doi-asserted-by":"crossref","DOI":"10.1142\/3573","article-title":"Regression and Time Series Model Selection","author":"AD McQuarrie","year":"1998"},{"key":"ref19","article-title":"A Note on the Validity of Cross-Validation for Evaluating Time Series Prediction","volume":"10","author":"C Bergmeir","year":"2015","journal-title":"Monash University, Department of Econometrics and Business Statistics, Working Paper"},{"key":"ref20","doi-asserted-by":"crossref","unstructured":"Bergmeir C, Ben\u00edtez JM. Forecaster performance evaluation with cross-validation and variants. In: Proc. 11th Intl. Conference on Intelligent Systems Design and Applications (ISDA). IEEE; 2011. p. 849\u2013854.","DOI":"10.1109\/ISDA.2011.6121763"},{"key":"ref21","doi-asserted-by":"crossref","first-page":"132","DOI":"10.1016\/j.csda.2014.02.001","article-title":"On the usefulness of cross-validation for directional forecast evaluation","volume":"76","author":"C Bergmeir","year":"2014","journal-title":"Computational Statistics & Data Analysis"},{"key":"ref22","doi-asserted-by":"crossref","unstructured":"Cerqueira V, Torgo L, Smailovi\u0107 J, Mozeti\u010d I. A comparative study of performance estimation methods for time series forecasting. In: Proc. 4th Intl. Conference on Data Science and Advanced Analytics (DSAA). IEEE; 2017. p. 529\u2013538.","DOI":"10.1109\/DSAA.2017.7"},{"issue":"5","key":"ref23","doi-asserted-by":"crossref","first-page":"e0155036","DOI":"10.1371\/journal.pone.0155036","article-title":"Multilingual Twitter sentiment classification: the role of human annotators","volume":"11","author":"I Mozeti\u010d","year":"2016","journal-title":"PLoS ONE"},{"key":"ref24","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4757-2440-0","article-title":"The Nature of Statistical Learning Theory","author":"VN Vapnik","year":"1995"},{"key":"ref25","first-page":"207","article-title":"Advances in Artificial Intelligence","author":"L Gaudette","year":"2009"},{"key":"ref26","doi-asserted-by":"crossref","unstructured":"Martineau J, Finin T. Delta TFIDF: An improved feature space for sentiment analysis. In: Proc. 3rd AAAI Intl. Conference on Weblogs and Social Media (ICWSM); 2009. p. 258\u2013261.","DOI":"10.1609\/icwsm.v3i1.13979"},{"key":"ref27","article-title":"Content Analysis, An Introduction to Its Methodology","author":"K Krippendorff","year":"2013"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"723","DOI":"10.1613\/jair.4272","article-title":"Sentiment analysis of short informal texts","volume":"50","author":"S Kiritchenko","year":"2014","journal-title":"Journal of Artificial Intelligence Research"},{"key":"ref29","article-title":"Information Retrieval","author":"CJ Van Rijsbergen","year":"1979"},{"issue":"200","key":"ref30","doi-asserted-by":"crossref","first-page":"675","DOI":"10.1080\/01621459.1937.10503522","article-title":"The use of ranks to avoid the assumption of normality implicit in the analysis of variance","volume":"32","author":"M Friedman","year":"1937","journal-title":"Journal of the American Statistical Association"},{"issue":"1","key":"ref31","doi-asserted-by":"crossref","first-page":"86","DOI":"10.1214\/aoms\/1177731944","article-title":"A comparison of alternative tests of significance for the problem of m rankings","volume":"11","author":"M Friedman","year":"1940","journal-title":"The Annals of Mathematical Statistics"},{"issue":"6","key":"ref32","doi-asserted-by":"crossref","first-page":"571","DOI":"10.1080\/03610928008827904","article-title":"Approximations of the critical region of the Friedman statistic","volume":"9","author":"RL Iman","year":"1980","journal-title":"Communications in Statistics-Theory and Methods"},{"issue":"Jan","key":"ref33","first-page":"1","article-title":"Statistical comparisons of classifiers over multiple data sets","volume":"7","author":"J Dem\u0161ar","year":"2006","journal-title":"Journal of Machine Learning Research"},{"key":"ref34","unstructured":"Nemenyi PB. Distribution-free Multiple Comparisons. PhD thesis, Princeton University, USA; 1963."},{"issue":"6","key":"ref35","doi-asserted-by":"crossref","first-page":"80","DOI":"10.2307\/3001968","article-title":"Individual comparisons by ranking methods","volume":"1","author":"F Wilcoxon","year":"1945","journal-title":"Biometrics Bulletin"},{"key":"ref36","unstructured":"Gr\u010dar M. Mining text-enriched heterogeneous information networks. PhD thesis, Jozef Stefan International Postgraduate School, Ljubljana, Slovenia; 2015."},{"issue":"1","key":"ref37","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1016\/S0304-4076(00)00030-0","article-title":"Consistent cross-validatory model-selection for dependent data: hv-block cross-validation","volume":"99","author":"J Racine","year":"2000","journal-title":"Journal of Econometrics"}],"container-title":["PLOS ONE"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pone.0194317","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,1]],"date-time":"2023-09-01T15:51:56Z","timestamp":1693583516000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pone.0194317"}},"subtitle":[],"editor":[{"given":"Frank","family":"Emmert-Streib","sequence":"first","affiliation":[]}],"short-title":[],"issued":{"date-parts":[[2018,3,13]]},"references-count":37,"journal-issue":{"issue":"3","published-online":{"date-parts":[[2018,3,13]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pone.0194317","relation":{},"ISSN":["1932-6203"],"issn-type":[{"value":"1932-6203","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,3,13]]}}}