{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,5]],"date-time":"2025-11-05T06:59:28Z","timestamp":1762325968100,"version":"3.37.3"},"reference-count":61,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T00:00:00Z","timestamp":1721865600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T00:00:00Z","timestamp":1721865600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,9]]},"DOI":"10.1007\/s10664-024-10518-9","type":"journal-article","created":{"date-parts":[[2024,7,25]],"date-time":"2024-07-25T08:01:54Z","timestamp":1721894514000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["The impact of concept drift and data leakage on log level prediction models"],"prefix":"10.1007","volume":"29","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8063-6225","authenticated-orcid":false,"given":"Youssef Esseddiq","family":"Ouatiti","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mohammed","family":"Sayagh","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Noureddine","family":"Kerzazi","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bram","family":"Adams","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ahmed E.","family":"Hassan","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,7,25]]},"reference":[{"key":"10518_CR1","unstructured":"Agrawal A,\u00a0Menzies T (2019) Is ai different for se? NC State Univ. 12"},{"key":"10518_CR2","doi-asserted-by":"crossref","unstructured":"Anu H,\u00a0Chen J,\u00a0Shi W,\u00a0Hou J,\u00a0Liang B,\u00a0Qin B (2019) An approach to recommendation of verbosity log levels based on logging intention. In: 2019 IEEE International Conference on Software Maintenance and Evolution (ICSME)","DOI":"10.1109\/ICSME.2019.00022"},{"key":"10518_CR3","doi-asserted-by":"crossref","unstructured":"Bennin KE, bin Ali N, B\u00f6rstler J, Yu X (2020) Revisiting the impact of concept drift on just-in-time quality assurance. (2020) IEEE 20th Int. Conf. on Software Quality, Reliability and Security (QRS)","DOI":"10.1109\/QRS51102.2020.00020"},{"key":"10518_CR4","unstructured":"Bertram T,\u00a0F\u00fcrnkranz J,\u00a0M\u00fcller M (2022) Quantity vs quality: investigating the trade-off between sample size and label reliability"},{"issue":"71","key":"10518_CR5","doi-asserted-by":"publisher","first-page":"147","DOI":"10.19101\/IJATEE.2020.762057","volume":"7","author":"R Chahar","year":"2020","unstructured":"Chahar R, Kaur D (2020) A systematic review of the machine learning algorithms for the computational analysis in different domains. International Journal of Advanced Technology and Engineering Exploration 7(71):147","journal-title":"International Journal of Advanced Technology and Engineering Exploration"},{"key":"10518_CR6","unstructured":"Chakkrit Tantithamthavorn (2018) ScottKnottESD: The Scott-Knott Effect Size Difference (ESD) Test"},{"issue":"4","key":"10518_CR7","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1109\/MCI.2015.2471196","volume":"10","author":"G Ditzler","year":"2015","unstructured":"Ditzler G, Roveri M, Alippi C, Polikar R (2015) Learning in nonstationary environments: a survey. IEEE Comput Intell Mag 10(4):12\u201325","journal-title":"IEEE Comput Intell Mag"},{"key":"10518_CR8","doi-asserted-by":"crossref","unstructured":"Ekanayake JB,\u00a0Tappolet J, Gall HC,\u00a0Bernstein A (2009) Tracking concept drift of software projects using defect prediction quality. 2009 6th IEEE Int. Working Conference on Mining Software Repositories, pages 51\u201360","DOI":"10.1109\/MSR.2009.5069480"},{"key":"10518_CR9","doi-asserted-by":"crossref","unstructured":"Fu Q,\u00a0Zhu J,\u00a0Hu W, Lou J-G,\u00a0Ding R,\u00a0Lin Q,\u00a0Zhang D,\u00a0Xie T (2014) Where do developers log? an empirical study on logging practices in industry. In: Proceedings of the 36th International Conference on Software Engineering(ICSE\u201914), page 24\u201333","DOI":"10.1145\/2591062.2591175"},{"key":"10518_CR10","first-page":"286","volume":"8","author":"J Gama","year":"2004","unstructured":"Gama J, Medas P, Castillo G, Rodrigues P (2004) Learning with drift detection. 8:286\u2013295","journal-title":"Learning with drift detection."},{"issue":"4","key":"10518_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2523813","volume":"46","author":"JA Gama","year":"2014","unstructured":"Gama JA, \u017dliobaitundefined I, Bifet A, Pechenizkiy M, Bouchachia A (2014) A survey on concept drift adaptation. ACM Comput Surv 46(4):1\u201337","journal-title":"ACM Comput Surv"},{"issue":"2","key":"10518_CR12","doi-asserted-by":"publisher","first-page":"171","DOI":"10.1023\/A:1010920819831","volume":"45","author":"DJ Hand","year":"2001","unstructured":"Hand DJ, Till RJ (2001) A simple generalisation of the area under the roc curve for multiple class classification problems. Mach Learn 45(2):171\u2013186","journal-title":"Mach Learn"},{"key":"10518_CR13","unstructured":"Harel M,\u00a0Crammer K,\u00a0El-Yaniv R,\u00a0Mannor S (2014) Concept drift detection through resampling. In: Proceedings of the 31st international conference on international conference on machine learning - vol 32, ICML\u201914, page II-1009-II-1017. JMLR.org"},{"key":"10518_CR14","doi-asserted-by":"crossref","unstructured":"Harrell FE (2001) Regression Modeling Strategies. Springer International Publishing","DOI":"10.1007\/978-1-4757-3462-1"},{"key":"10518_CR15","doi-asserted-by":"crossref","unstructured":"Haynes W (2013) Bonferroni Correction, pages 154\u2013154. Springer New York","DOI":"10.1007\/978-1-4419-9863-7_1213"},{"key":"10518_CR16","doi-asserted-by":"crossref","unstructured":"Herbsleb J,\u00a0Mockus A (2003) An empirical study of speed and communication in globally distributed software development. IEEE Transactions on Software Engineering","DOI":"10.1109\/TSE.2003.1205177"},{"key":"10518_CR17","doi-asserted-by":"crossref","unstructured":"Jia Z,\u00a0Li S,\u00a0Liu X,\u00a0Liao X,\u00a0Liu Y (2018) Smartlog: place error log statement by deep understanding of log intention. In: 2018 IEEE 25th international conference on Software Analysis, Evolution and Reengineering (SANER), pages 61\u201371","DOI":"10.1109\/SANER.2018.8330197"},{"key":"10518_CR18","unstructured":"Jiarpakdee J,\u00a0Tantithamthavorn C, Hassan AE (2018) The impact of correlated metrics on defect models. arXiv:1801.10271"},{"key":"10518_CR19","doi-asserted-by":"crossref","unstructured":"Kabinna S,\u00a0Shang W,\u00a0Bezemer C, Hassan AE (2016) Examining the stabity of logging statements. In: Proceedings of the 23rd international conference on Software Analysis, Evolution, and Reengineering (SANER), vol 1, pages 326\u2013337","DOI":"10.1109\/SANER.2016.29"},{"key":"10518_CR20","doi-asserted-by":"crossref","unstructured":"Kaufman S,\u00a0Rosset S,\u00a0Perlich C (2011) Leakage in data mining: formulation, detection, and avoidance. vol 6, pages 556\u2013563","DOI":"10.1145\/2020408.2020496"},{"key":"10518_CR21","unstructured":"Lee D, Rajbahadur GK,\u00a0Lin D,\u00a0Sayagh M, Bezemer C-P, Hassan AE (2019) An empirical study of the characteristics of popular minecraft mods. Empirical Software Engineering, pages 1\u201323"},{"key":"10518_CR22","doi-asserted-by":"crossref","unstructured":"Lee D, Rajbahadur GK,\u00a0Lin D,\u00a0Sayagh M, Bezemer C-P, Hassan AE (2020) An empirical study of the characteristics of popular minecraft mods. Empir Softw Eng","DOI":"10.1007\/s10664-020-09840-9"},{"issue":"5","key":"10518_CR23","doi-asserted-by":"publisher","first-page":"2655","DOI":"10.1007\/s10664-018-9595-8","volume":"23","author":"H Li","year":"2018","unstructured":"Li H, Chen T-HP, Shang W, Hassan AE (2018) Studying software logging using topic models. Empirical Software Enggineering 23(5):2655\u20132694","journal-title":"Empirical Software Enggineering"},{"issue":"5","key":"10518_CR24","doi-asserted-by":"publisher","first-page":"2655","DOI":"10.1007\/s10664-018-9595-8","volume":"23","author":"H Li","year":"2018","unstructured":"Li H, Chen T-HP, Shang W, Hassan AE (2018) Studying software logging using topic models. Empirical Softw. Eng. 23(5):2655\u20132694","journal-title":"Empirical Softw. Eng."},{"key":"10518_CR25","unstructured":"Li H,\u00a0Shang W,\u00a0Adams B,\u00a0Sayagh M, Hassan AE (2020) A qualitative study of the benefits and costs of logging from developers\u2019 perspectives. IEEE Transactions on Software Engineering, pages 1\u20131"},{"key":"10518_CR26","doi-asserted-by":"crossref","unstructured":"Li H,\u00a0Shang W,\u00a0Hassan A (2017) Which log level should developers choose for a new logging statement? Empirical Software Engineering, page 1684\u20131716","DOI":"10.1007\/s10664-016-9456-2"},{"key":"10518_CR27","doi-asserted-by":"crossref","unstructured":"Li Z,\u00a0Li H, Chen T-H,\u00a0Shang W (2021) Deeplv: suggesting log levels using ordinal based neural networks. In: 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), pages 1461\u20131472","DOI":"10.1109\/ICSE43902.2021.00131"},{"key":"10518_CR28","doi-asserted-by":"crossref","unstructured":"Lin Q,\u00a0Hsieh K,\u00a0Dang Y,\u00a0Zhang H,\u00a0Sui K,\u00a0Xu Y, Lou J-G,\u00a0Li C,\u00a0Wu Y, Yao R,\u00a0Chintalapati M,\u00a0Zhang D (2018) Predicting node failure in cloud service systems. In: Proceedings of the 2018 26th ACM joint meeting on European software engineering conference and symposium on the foundations of software engineering, ESEC\/FSE 2018, page 480\u2013490, New York, USA. Association for Computing Machinery","DOI":"10.1145\/3236024.3236060"},{"key":"10518_CR29","doi-asserted-by":"crossref","unstructured":"Liu A, SongY, Zhang G,\u00a0Lu J (2017) Regional concept drift detection and density synchronized drift adaptation. In: Proceedings of the 26th International Joint Conference on Artificial Intelligence, IJCAI\u201917","DOI":"10.24963\/ijcai.2017\/317"},{"key":"10518_CR30","doi-asserted-by":"crossref","unstructured":"Liu Z,\u00a0Xia X,\u00a0Lo D,\u00a0Xing Z, Hassan AE,\u00a0Li S (2019) Which variables should i log? IEEE Transactions on Software Engineering, pages 1\u20131","DOI":"10.1109\/TSE.2019.2941943"},{"key":"10518_CR31","doi-asserted-by":"crossref","unstructured":"Lu N, Lu J, Zhang G, Lopez de Mantaras R (2016) A concept drift-tolerant case-base editing technique. Artif Intell 230(C):108\u2013133","DOI":"10.1016\/j.artint.2015.09.009"},{"issue":"12","key":"10518_CR32","first-page":"2346","volume":"31","author":"J Lu","year":"2019","unstructured":"Lu J, Liu A, Dong F, Gu F, Gama J, Zhang G (2019) Learning under concept drift: a review. IEEE Trans Knowl Data Eng 31(12):2346\u20132363","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"10518_CR33","doi-asserted-by":"crossref","unstructured":"Lyu Y,\u00a0Li H,\u00a0Sayagh M,\u00a0Jiang Z, Hassan AE (2021) An empirical study of the impact of data splitting decisions on the performance of aiops solutions. ACM Transactions on Software Engineering and Methodology","DOI":"10.1145\/3447876"},{"key":"10518_CR34","doi-asserted-by":"crossref","unstructured":"Lyu Y, Rajbahadur GK,\u00a0Lin D,\u00a0Chen B, Jiang ZMJ (2021) Towards a consistent interpretation of aiops models. 31(1)","DOI":"10.1145\/3488269"},{"key":"10518_CR35","doi-asserted-by":"crossref","unstructured":"McIntosh BAS,\u00a0Kamei Y, Hassan AE (2014) The impact of code review coverage and code review participation on software quality. In: Proceedings of the working conference on Mining Software Repositories (MSR), page 292\u2013201","DOI":"10.1145\/2597073.2597076"},{"key":"10518_CR36","doi-asserted-by":"crossref","unstructured":"Olewicki D, Nayrolles M, Adams B (2022) Towards language-independent brown build detection. In: Proc. of the 44th Int. Conf. on Software Engineering(ICSE\u201922)","DOI":"10.1145\/3510003.3510122"},{"key":"10518_CR37","doi-asserted-by":"crossref","unstructured":"Oliner A,\u00a0Ganapathi A,\u00a0Xu W (2012) Advances and challenges in log analysis. Communications of the ACM, page 55\u201361","DOI":"10.1145\/2076450.2076466"},{"key":"10518_CR38","doi-asserted-by":"crossref","unstructured":"Ouatiti YE (2024) The impact of concept drift and data leakage on log level prediction models - appendix. https:\/\/zenodo.org\/records\/10898284","DOI":"10.1007\/s10664-024-10518-9"},{"key":"10518_CR39","doi-asserted-by":"crossref","unstructured":"Ouatiti YE,\u00a0Sayagh M,\u00a0Kerzazi N, Hassan AE (2023) An empirical study on log level prediction for multi-component systems. IEEE Trans Softw Eng","DOI":"10.1109\/TSE.2022.3154672"},{"key":"10518_CR40","doi-asserted-by":"crossref","unstructured":"Pecchia A,\u00a0Cinque M,\u00a0Carrozza G,\u00a0Cotroneo D (2015) Industry practices and event logging: assessment of a critical software development process. In: Proceedings of the 37th international conference on software engineering, vol 2, pages 169\u2013178","DOI":"10.1109\/ICSE.2015.145"},{"key":"10518_CR41","unstructured":"Rajbahadur GK,\u00a0Wang S,\u00a0Ansaldi G,\u00a0Kamei Y, Hassan AE (2021) The impact of feature importance methods on the interpretation of defect classifiers. IEEE Transactions on Software Engineering, pages 1\u20131"},{"key":"10518_CR42","doi-asserted-by":"crossref","unstructured":"Ramrez-Gallego S,\u00a0Krawczyk B,\u00a0Garca S,\u00a0Woniak M,\u00a0Herrera F (2017) A survey on data preprocessing for data stream mining. Neurocomput 239(C):39\u201357","DOI":"10.1016\/j.neucom.2017.01.078"},{"key":"10518_CR43","unstructured":"rms: Regression Modeling Strategies"},{"key":"10518_CR44","doi-asserted-by":"crossref","unstructured":"Sarro F,\u00a0Moussa R,\u00a0Petrozziello A,\u00a0Harman M (2022) Learning from mistakes: machine learning enhanced human expert effort estimates. IEEE Trans Softw Eng","DOI":"10.1109\/TSE.2020.3040793"},{"key":"10518_CR45","doi-asserted-by":"crossref","unstructured":"Sayagh M,\u00a0Dong Z,\u00a0Andrzejak A,\u00a0Adams B (2017) Does the choice of configuration framework matter for developers? empirical study on 11 java configuration frameworks. In: 2017 IEEE 17th international working conference on Source Code Analysis and Manipulation (SCAM)","DOI":"10.1109\/SCAM.2017.25"},{"issue":"1","key":"10518_CR46","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10664-013-9274-8","volume":"20","author":"W Shang","year":"2015","unstructured":"Shang W, Nagappan M, Hassan AE (2015) Studying the relationship between logging characteristics and the code quality of platform software. Empirical Software Enggineering 20(1):1\u201327","journal-title":"Empirical Software Enggineering"},{"key":"10518_CR47","doi-asserted-by":"crossref","unstructured":"Shihab E,\u00a0Kamei Y,\u00a0Adams B, Hassan AE (2013) Is lines of code a good measure of effort in effort-aware models? Inf Softw Technol","DOI":"10.1016\/j.infsof.2013.06.002"},{"key":"10518_CR48","doi-asserted-by":"crossref","unstructured":"Subramanyam R,\u00a0Krishnan M (2003) Empirical analysis of ck metrics for object-oriented design complexity: implications for software defects. IEEE Trans Softw Eng","DOI":"10.1109\/TSE.2003.1191795"},{"key":"10518_CR49","doi-asserted-by":"crossref","unstructured":"Tantithamthavorn C, Hassan AE (2018) An experience report on defect modelling in practice: pitfalls and challenges. In: Proceedings of the International Conference on Software Engineering: Software Engineering in Practice Track (ICSE-SEIP\u201918), page To Appear","DOI":"10.1145\/3183519.3183547"},{"key":"10518_CR50","unstructured":"Tantithamthavorn C, Hassan AE,\u00a0Matsumoto K (2018) The impact of class rebalancing techniques on the performance and interpretation of defect prediction models"},{"key":"10518_CR51","doi-asserted-by":"crossref","unstructured":"Tantithamthavorn C,\u00a0Jiarpakdee J,\u00a0Grundy J (2021) Actionable analytics: stop telling me what it is; please tell me what to do. IEEE Software","DOI":"10.1109\/MS.2021.3072088"},{"issue":"07","key":"10518_CR52","doi-asserted-by":"publisher","first-page":"683","DOI":"10.1109\/TSE.2018.2794977","volume":"45","author":"C Tantithamthavorn","year":"2019","unstructured":"Tantithamthavorn C, McIntosh S, Hassan AE, Matsumoto K (2019) The impact of automated parameter optimization on defect prediction models. IEEE Trans Software Eng 45(07):683\u2013711","journal-title":"IEEE Trans Software Eng"},{"key":"10518_CR53","unstructured":"Thongtanunam P,\u00a0Hassan A (2018) Review dynamics and its impact on software quality. IEEE Transactions on Software Engineering, pages 1\u201313"},{"key":"10518_CR54","unstructured":"Yuan D, Luo Y,\u00a0Zhuang X, Rodrigues GR,\u00a0Zhao X,\u00a0Zhang Y, Jain PU,\u00a0Stumm M (2014) Simple testing can prevent most critical failures: an analysis of production failures in distributed data-intensive systems. In: Proceedings of the 11th conference on operating systems design and implementation, systems design and implementation, pages 249\u2013265"},{"key":"10518_CR55","unstructured":"Yuan D,\u00a0Park S,\u00a0Huang P,\u00a0Liu Y, Lee MM,\u00a0Tang X,\u00a0Zhou Y,\u00a0Savage S (2012) Be conservative: enhancing failure diagnosis with proactive logging. In: Proceedings of the 10th USENIX conference on Operating Systems Design and Implementation, OSDI\u201912"},{"key":"10518_CR56","doi-asserted-by":"crossref","unstructured":"Yuan D,\u00a0Park S,\u00a0Zhou Y (2012) Characterizing logging practices in open-source software. In: Proceedings of the 34th International Conference on Software Engineering(ICSE\u201912), pages 102\u2013112","DOI":"10.1109\/ICSE.2012.6227202"},{"key":"10518_CR57","doi-asserted-by":"crossref","unstructured":"Yuan D,\u00a0Zheng J,\u00a0Park S,\u00a0Zhou Y,\u00a0Savage S (2011) Improving software diagnosability via log enhancement. In: Proceedings of the sixteenth international conference on architectural support for programming languages and operating systems, page 3\u201314. Association for Computing Machinery","DOI":"10.1145\/1950365.1950369"},{"key":"10518_CR58","unstructured":"Zhang D,\u00a0Tsai J (2002) Machine learning and software engineering"},{"key":"10518_CR59","doi-asserted-by":"crossref","unstructured":"Zhang J, Sheng VS,\u00a0Wu J,\u00a0Wu X (2016) Multi-class ground truth inference in crowdsourcing with clustering. IEEE Transactions on Knowledge and Data Engineering","DOI":"10.1109\/TKDE.2015.2504974"},{"key":"10518_CR60","doi-asserted-by":"crossref","unstructured":"Zhou Z-H (2012) Ensemble Methods: Foundations and Algorithms. Chapman & Hall\/CRC","DOI":"10.1201\/b12207"},{"key":"10518_CR61","doi-asserted-by":"crossref","unstructured":"Zhu J,\u00a0He P,\u00a0Fu Q,\u00a0Zhang H, Lyu MR,\u00a0Zhang D (2015) Learning to log: Helping developers make informed logging decisions. In: Proceedings of the 37th International Conference on Software Engineering - vol 1, ICSE\u201915, pages 415\u2013425","DOI":"10.1109\/ICSE.2015.60"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10518-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-024-10518-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-024-10518-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,6]],"date-time":"2024-09-06T03:52:00Z","timestamp":1725594720000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-024-10518-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,25]]},"references-count":61,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2024,9]]}},"alternative-id":["10518"],"URL":"https:\/\/doi.org\/10.1007\/s10664-024-10518-9","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"type":"print","value":"1382-3256"},{"type":"electronic","value":"1573-7616"}],"subject":[],"published":{"date-parts":[[2024,7,25]]},"assertion":[{"value":"17 June 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 July 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of Interest"}}],"article-number":"123"}}