{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,5]],"date-time":"2026-02-05T11:11:07Z","timestamp":1770289867341,"version":"3.49.0"},"reference-count":44,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2016,10,14]],"date-time":"2016-10-14T00:00:00Z","timestamp":1476403200000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2017,8]]},"DOI":"10.1007\/s10664-016-9456-2","type":"journal-article","created":{"date-parts":[[2016,10,14]],"date-time":"2016-10-14T04:43:54Z","timestamp":1476420234000},"page":"1684-1716","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":84,"title":["Which log level should developers choose for a new logging statement?"],"prefix":"10.1007","volume":"22","author":[{"given":"Heng","family":"Li","sequence":"first","affiliation":[]},{"given":"Weiyi","family":"Shang","sequence":"additional","affiliation":[]},{"given":"Ahmed E.","family":"Hassan","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2016,10,14]]},"reference":[{"key":"9456_CR1","unstructured":"Aguinis H (2004) Regression analysis for categorical moderators. Guilford Press"},{"issue":"1","key":"9456_CR2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1175\/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2","volume":"78","author":"GW Brier","year":"1950","unstructured":"Brier GW (1950) Verification of forecasts expressed in terms of probability. Mon Weather Rev 78(1):1\u20133","journal-title":"Mon Weather Rev"},{"key":"9456_CR3","doi-asserted-by":"crossref","unstructured":"Cohen J, Cohen P, West SG, Aiken LS (2013) Applied multiple regression\/correlation analysis for the behavioral sciences. Routledge","DOI":"10.4324\/9780203774441"},{"key":"9456_CR4","unstructured":"Cullmann AD (2015) HandTill2001: Multiple Class Area under ROC Curve. R package version 0.2-10."},{"issue":"4-5","key":"9456_CR5","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1007\/s10664-011-9173-9","volume":"17","author":"M D\u2019Ambros","year":"2012","unstructured":"D\u2019Ambros M, Lanza M, Robbes R (2012) Evaluating defect prediction approaches: a benchmark and an extensive comparison. Empir Softw Eng 17(4-5):531\u2013577","journal-title":"Empir Softw Eng"},{"key":"9456_CR6","unstructured":"Eberhardt C (2014) The art of logging. http:\/\/www.codeproject.com\/Articles\/42354\/The-Art-of-Logging . Accessed 12 May 2016"},{"issue":"1","key":"9456_CR7","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1214\/aos\/1176344552","volume":"7","author":"B Efron","year":"1979","unstructured":"Efron B (1979) Bootstrap methods: another look at the jackknife. Ann Stat 7 (1):1\u201326","journal-title":"Ann Stat"},{"issue":"394","key":"9456_CR8","doi-asserted-by":"crossref","first-page":"461","DOI":"10.1080\/01621459.1986.10478291","volume":"81","author":"B Efron","year":"1986","unstructured":"Efron B (1986) How biased is the apparent error rate of a prediction rule? J Am Stat Assoc 81(394):461\u2013 470","journal-title":"J Am Stat Assoc"},{"key":"9456_CR9","doi-asserted-by":"crossref","unstructured":"Fu Q, Zhu J, Hu W, Lou J-G, Ding R, Lin Q, Zhang D, Xie T (2014) Where do developers log? An empirical study on logging practices in industry. In: Companion Proceedings of the 36th International Conference on Software Engineering, ICSE Companion \u201914, pp 24\u201333","DOI":"10.1145\/2591062.2591175"},{"issue":"7","key":"9456_CR10","doi-asserted-by":"crossref","first-page":"653","DOI":"10.1109\/32.859533","volume":"26","author":"TL Graves","year":"2000","unstructured":"Graves TL, Karr AF, Marron JS, Siy H (2000) Predicting fault incidence using software change history. IEEE Trans Softw Eng 26(7):653\u2013661","journal-title":"IEEE Trans Softw Eng"},{"key":"9456_CR11","unstructured":"G\u00fclc\u00fc C, Stark S (2003) The complete log4j manual. Quality Open Software"},{"issue":"2","key":"9456_CR12","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1023\/A:1010920819831","volume":"45","author":"DJ Hand","year":"2001","unstructured":"Hand DJ, Till RJ (2001) A simple generalisation of the area under the ROC curve for multiple class classification problems. Mach Learn 45(2):171\u2013186","journal-title":"Mach Learn"},{"key":"9456_CR13","doi-asserted-by":"crossref","unstructured":"Harrell Jr FE (2015a) Regression modeling strategies: with applications to linear models, logistic and ordinal regression, and survival analysis. Springer","DOI":"10.1007\/978-3-319-19425-7"},{"key":"9456_CR14","unstructured":"Harrell Jr FE (2015b) rms: Regression Modeling Strategies. R package version 4.4-1"},{"key":"9456_CR15","unstructured":"Harrell Jr FE (2014) with contributions from Charles Dupont, and many others. Hmisc: Harrell Miscellaneous. R package version 3.14-5"},{"key":"9456_CR16","doi-asserted-by":"crossref","unstructured":"Hassan AE (2009) Predicting faults using the complexity of code changes. In: Proceedings of the 31st International Conference on Software Engineering, ICSE \u201909, pp 78\u201388","DOI":"10.1109\/ICSE.2009.5070510"},{"key":"9456_CR17","doi-asserted-by":"crossref","unstructured":"Kabinna S, Shang W, Bezemer C-P, Hassan AE (2016a) Examining the stability of logging statements. In: Proceedings of the 23rd IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER \u201916, pp 326\u2013337","DOI":"10.1109\/SANER.2016.29"},{"key":"9456_CR18","doi-asserted-by":"crossref","unstructured":"Kabinna S, Bezemer C-P, Shang W, Hassan AE (2016b) Logging library migrations: a case study for the apache software foundation projects. In: Proceedings of the 13th International Conference on Mining Software Repositories, MSR \u201916, pp 154\u2013164","DOI":"10.1145\/2901739.2901769"},{"key":"9456_CR19","doi-asserted-by":"crossref","unstructured":"Kuhn M, Johnson K (2013) Applied predictive modeling. Springer","DOI":"10.1007\/978-1-4614-6849-3"},{"issue":"2","key":"9456_CR20","doi-asserted-by":"crossref","first-page":"318","DOI":"10.2307\/2530022","volume":"34","author":"J Lawless","year":"1978","unstructured":"Lawless J, Singhal K (1978) Efficient screening of nonnormal regression models. Biometrics 34(2):318\u2013327","journal-title":"Biometrics"},{"issue":"32","key":"9456_CR21","doi-asserted-by":"crossref","first-page":"1","DOI":"10.3310\/hta13320","volume":"13","author":"J Mant","year":"2009","unstructured":"Mant J, Doust J, Roalfe A, Barton P, Cowie MR, Glasziou P, Mant D, McManus R, Holder R, Deeks J et al (2009) Systematic review and individual patient data meta-analysis of diagnosis of heart failure, with modelling of implications of different diagnostic strategies in primary care. Health Technol Assess 13(32):1\u2013207","journal-title":"Health Technol Assess"},{"issue":"3","key":"9456_CR22","doi-asserted-by":"crossref","first-page":"621","DOI":"10.1080\/00401706.1970.10488701","volume":"12","author":"N Mantel","year":"1970","unstructured":"Mantel N (1970) Why stepdown procedures in variable selection. Technometrics 12(3):621\u2013625","journal-title":"Technometrics"},{"key":"9456_CR23","doi-asserted-by":"crossref","unstructured":"Mariani L, Pastore F (2008) Automated identification of failure causes in system logs. In: Proceedings of the 2008 19th International Symposium on Software Reliability Engineering, ISSRE \u201908, pp 117\u2013126","DOI":"10.1109\/ISSRE.2008.48"},{"key":"9456_CR24","doi-asserted-by":"crossref","unstructured":"Mariani L, Pastore F, Pezze M (2009) A toolset for automated failure analysis. In: Proceedings of the 31st International Conference on Software Engineering, ICSE \u201909, pp 563\u2013566","DOI":"10.1109\/ICSE.2009.5070556"},{"issue":"2","key":"9456_CR25","doi-asserted-by":"crossref","first-page":"109","DOI":"10.1111\/j.2517-6161.1980.tb01109.x","volume":"42","author":"P McCullagh","year":"1980","unstructured":"McCullagh P (1980) Regression models for ordinal data. J Royal Stat Soc. Ser B (Methodological) 42(2):109\u2013142","journal-title":"J Royal Stat Soc. Ser B (Methodological)"},{"key":"9456_CR26","doi-asserted-by":"crossref","unstructured":"McIntosh S, Kamei Y, Adams B, Hassan AE (2014) The impact of code review coverage and code review participation on software quality: A case study of the Qt, VTK, and ITK projects. In: Proceedings of the 11th Working Conference on Mining Software Repositories, MSR 2014, pp 192\u2013201","DOI":"10.1145\/2597073.2597076"},{"issue":"5","key":"9456_CR27","doi-asserted-by":"crossref","first-page":"2146","DOI":"10.1007\/s10664-015-9381-9","volume":"21","author":"S McIntosh","year":"2016","unstructured":"McIntosh S, Kamei Y, Adams B, Hassan AE (2016) An empirical study of the impact of modern code review practices on software quality. Empir Softw Eng 21 (5):2146\u20132189","journal-title":"Empir Softw Eng"},{"issue":"1","key":"9456_CR28","doi-asserted-by":"crossref","first-page":"103","DOI":"10.1080\/0022250X.1975.9989847","volume":"4","author":"RD McKelvey","year":"1975","unstructured":"McKelvey RD, Zavoina W (1975) A statistical model for the analysis of ordinal level dependent variables. J Math Sociol 4(1):103\u2013120","journal-title":"J Math Sociol"},{"key":"9456_CR29","unstructured":"MSDN (2011) Logging an exception. https:\/\/msdn.microsoft.com\/en-us\/library\/ff664711(v=pandp.50).aspx . Accessed 12 May 2016"},{"issue":"2","key":"9456_CR30","doi-asserted-by":"crossref","first-page":"55","DOI":"10.1145\/2076450.2076466","volume":"55","author":"A Oliner","year":"2012","unstructured":"Oliner A, Ganapathi A, Xu W (2012) Advances and challenges in log analysis. Commun ACM 55(2):55\u201361","journal-title":"Commun ACM"},{"key":"9456_CR31","doi-asserted-by":"crossref","unstructured":"Pecchia A, Cinque M, Carrozza G, Cotroneo D (2015) Industry practices and event logging: Assessment of a critical software development process. In: Proceedings of the 37th International Conference on Software Engineering - Volume 2 , ICSE \u201915, pp 169\u2013178","DOI":"10.1109\/ICSE.2015.145"},{"key":"9456_CR32","doi-asserted-by":"crossref","unstructured":"Shang W, Jiang ZM, Adams B, Hassan AE, Godfrey MW, Nasser M, Flora P (2011) An exploratory study of the evolution of communicated information about the execution of large software systems. In: Proceedings of the 18th Working Conference on Reverse Engineering, WCRE \u201911, pp 335\u2013 344","DOI":"10.1109\/WCRE.2011.48"},{"issue":"1","key":"9456_CR33","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1002\/smr.1579","volume":"26","author":"W Shang","year":"2014","unstructured":"Shang W, Jiang ZM, Adams B, Hassan AE, Godfrey MW, Nasser M, Flora P (2014) An exploratory study of the evolution of communicated information about the execution of large software systems. J Softw Evol Process 26(1):3\u201326","journal-title":"J Softw Evol Process"},{"key":"9456_CR34","doi-asserted-by":"crossref","unstructured":"Shihab E, Jiang ZM, Ibrahim WM, Adams B, Hassan AE (2010) Understanding the impact of code and process metrics on postrelease defects: A case study on the Eclipse project. In: Proceedings of the 2010 ACM-IEEE International Symposium on Empirical Software Engineering and Measurement, ESEM \u201910, pp 4:1\u20134:10","DOI":"10.1145\/1852786.1852792"},{"issue":"1","key":"9456_CR35","doi-asserted-by":"crossref","first-page":"15","DOI":"10.2307\/2986219","volume":"45","author":"S Sommer","year":"1996","unstructured":"Sommer S, Huggins RM (1996) Variables selection using the Wald test and a robust CP. Appl Stat 45(1):15\u201329","journal-title":"Appl Stat"},{"key":"9456_CR36","unstructured":"Wilks DS (2011) Statistical methods in the atmospheric sciences, vol 100. Academic press"},{"key":"9456_CR37","doi-asserted-by":"crossref","unstructured":"Xu W, Huang L, Fox A, Patterson D, Jordan MI (2009) Detecting large-scale system problems by mining console logs. In: Proceedings of the ACM SIGOPS 22nd symposium on Operating systems principles, SOSP \u201909, pp 117\u2013132","DOI":"10.1145\/1629575.1629587"},{"key":"9456_CR38","doi-asserted-by":"crossref","unstructured":"Yuan D, Mai H, Xiong W, Tan L, Zhou Y, Pasupathy S (2010) Sherlog: Error diagnosis by connecting clues from run-time logs. In: Proceedings of the 15th International Conference on Architectural Support for Programming Languages and Operating Systems, ASPLOS XV, pp 143\u2013154","DOI":"10.1145\/1736020.1736038"},{"key":"9456_CR39","doi-asserted-by":"crossref","unstructured":"Yuan D, Zheng J, Park S, Zhou Y, Savage S (2011) Improving software diagnosability via log enhancement. In: Proceedings of the 16th International Conference on Architectural Support for Programming Languages and Operating Systems, ASPLOS XVI, pp 3\u201314","DOI":"10.1145\/1950365.1950369"},{"key":"9456_CR40","unstructured":"Yuan D, Park S, Huang P, Liu Y, Lee MM, Tang X, Zhou Y, Savage S (2012a) Be conservative: Enhancing failure diagnosis with proactive logging. In: Proceedings of the 10th USENIX conference on Operating Systems Design and Implementation, volume 12 of OSDI \u201912 , pp 293\u2013306"},{"key":"9456_CR41","doi-asserted-by":"crossref","unstructured":"Yuan D, Park S, Zhou Y (2012b) Characterizing logging practices in open-source software. In: Proceedings of the 34th International Conference on Software Engineering, ICSE \u201912, pp 102\u2013112","DOI":"10.1109\/ICSE.2012.6227202"},{"key":"9456_CR42","unstructured":"Yuan D, Luo Y, Zhuang X, Rodrigues GR, Zhao X, Zhang Y, Jain PU, Stumm M (2014) Simple testing can prevent most critical failures: An analysis of production failures in distributed data-intensive systems. In: Proceedings of the 11th USENIX Conference on Operating Systems Design and Implementation, OSDI \u201914, pp 249\u2013265"},{"key":"9456_CR43","doi-asserted-by":"crossref","unstructured":"Zhu J, He P, Fu Q, Zhang H, Lyu MR, Zhang D (2015) Learning to log: Helping developers make informed logging decisions. In: Proceedings of the 37th International Conference on Software Engineering - Volume 1 , ICSE \u201915, pp 415\u2013425","DOI":"10.1109\/ICSE.2015.60"},{"key":"9456_CR44","doi-asserted-by":"crossref","unstructured":"Zimmermann T, Weisgerber P, Diehl S, Zeller A (2004) Mining version histories to guide software changes. In: Proceedings of the 26th International Conference on Software Engineering, ICSE \u201904, pp 563\u2013572","DOI":"10.1109\/ICSE.2004.1317478"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-016-9456-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-016-9456-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-016-9456-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,19]],"date-time":"2024-06-19T22:54:09Z","timestamp":1718837649000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-016-9456-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,10,14]]},"references-count":44,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2017,8]]}},"alternative-id":["9456"],"URL":"https:\/\/doi.org\/10.1007\/s10664-016-9456-2","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,10,14]]}}}