{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,29]],"date-time":"2026-05-29T16:39:04Z","timestamp":1780072744805,"version":"3.54.0"},"reference-count":53,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2017,6,15]],"date-time":"2017-06-15T00:00:00Z","timestamp":1497484800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2018,2]]},"DOI":"10.1007\/s10664-017-9518-0","type":"journal-article","created":{"date-parts":[[2017,6,15]],"date-time":"2017-06-15T22:01:10Z","timestamp":1497564070000},"page":"290-333","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":81,"title":["Examining the stability of logging statements"],"prefix":"10.1007","volume":"23","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0784-7725","authenticated-orcid":false,"given":"Suhas","family":"Kabinna","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Cor-Paul","family":"Bezemer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weiyi","family":"Shang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mark D.","family":"Syer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmed E.","family":"Hassan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,6,15]]},"reference":[{"key":"9518_CR1","doi-asserted-by":"crossref","unstructured":"Bigliardi L, Lanza M, Bacchelli A, D\u2019Ambros M, Mocci A (2014) Quantitatively exploring non-code software artifacts. In: 14th international conference on quality software (QSIC), 2014. IEEE, pp 286\u2013 295","DOI":"10.1109\/QSIC.2014.31"},{"key":"9518_CR2","doi-asserted-by":"crossref","unstructured":"Bird C, Rigby PC, Barr ET, Hamilton DJ, German DM, Devanbu P (2009) The promises and perils of mining git. In: 6th IEEE international working conference on mining software repositories, 2009. MSR\u201909. IEEE, pp 1\u201310","DOI":"10.1109\/MSR.2009.5069475"},{"key":"9518_CR3","unstructured":"Boulon J, Konwinski A, Qi R, Rabkin A, Yang E, Yang M (2008) Chukwa, a large-scale monitoring system. In: Proceedings of cloud computing and its applications, vol 8, pp 1\u20135"},{"key":"9518_CR4","volume-title":"Exploring splunk","author":"D Carasso","year":"2012","unstructured":"Carasso D (2012) Exploring splunk. CITO Research, New York, USA. ISBN, p 978"},{"key":"9518_CR5","doi-asserted-by":"crossref","unstructured":"Cohen J, Cohen P, West S G, Aiken L S (2013) Applied multiple regression\/correlation analysis for the behavioral sciences. Routledge","DOI":"10.4324\/9780203774441"},{"key":"9518_CR6","doi-asserted-by":"crossref","unstructured":"Collett D (2015) Modelling survival data in medical research. CRC Press","DOI":"10.1201\/b18041"},{"key":"9518_CR7","unstructured":"Ding R, Zhou H, Lou J-G, Zhang H, Lin Q, Fu Q, Zhang D, Xie T (2015) Log2: a cost-aware logging mechanism for performance diagnosis. In: 2015 USENIX annual technical conference (USENIX ATC 15), pp 139\u2013150"},{"issue":"3","key":"9518_CR8","doi-asserted-by":"crossref","first-page":"403","DOI":"10.2307\/2297364","volume":"49","author":"C Elbers","year":"1982","unstructured":"Elbers C, Ridder G (1982) True and spurious duration dependence: the identifiability of the proportional hazard model. Rev Econ Stud 49(3):403\u2013409","journal-title":"Rev Econ Stud"},{"issue":"1","key":"9518_CR9","doi-asserted-by":"crossref","first-page":"145","DOI":"10.1146\/annurev.publhealth.20.1.145","volume":"20","author":"LD Fisher","year":"1999","unstructured":"Fisher L D, Lin D Y (1999) Time-dependent covariates in the cox proportional-hazards regression model. Annu Rev Public Health 20(1):145\u2013157","journal-title":"Annu Rev Public Health"},{"key":"9518_CR10","doi-asserted-by":"crossref","unstructured":"Fu Q, Zhu J, Hu W, Lou J-G, Ding R, Lin Q, Zhang D, Xie T (2014) Where do developers log? An empirical study on logging practices in industry. In: Proceedings of ICSE companion 2014: the 36th international conference on software engineering, pp 24\u201333","DOI":"10.1145\/2591062.2591175"},{"key":"9518_CR11","doi-asserted-by":"crossref","unstructured":"Fu Q, Lou J-G, Wang Y, Li J (2009) Execution anomaly detection in distributed systems through unstructured log analysis. In: Proceedings of the ICDM 2009, ninth IEEE international conference on data mining. IEEE, pp 149\u2013158","DOI":"10.1109\/ICDM.2009.60"},{"key":"9518_CR12","doi-asserted-by":"crossref","unstructured":"Ghotra B, McIntosh S, Hassan AE (2015) Revisiting the impact of classification techniques on the performance of defect prediction models. In: Proceedings of the 37th international conference on software engineering, vol 1. IEEE Press, pp 789\u2013800","DOI":"10.1109\/ICSE.2015.91"},{"key":"9518_CR13","unstructured":"Greenwood PE, Nikulin MS (1996) A guide to chi-squared testing, vol 280. Wiley"},{"key":"9518_CR14","doi-asserted-by":"crossref","unstructured":"Harrell F (2015) Regression modeling strategies: with applications to linear models, logistic and ordinal regression, and survival analysis. Springer","DOI":"10.1007\/978-3-319-19425-7"},{"issue":"2","key":"9518_CR15","first-page":"83","volume":"27","author":"T Hastie","year":"2005","unstructured":"Hastie T, Tibshirani R, Friedman J, Franklin J (2005) The elements of statistical learning: data mining, inference and prediction. Math Intell 27(2):83\u201385","journal-title":"Math Intell"},{"issue":"1","key":"9518_CR16","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1080\/00031305.1978.10479237","volume":"32","author":"DC Hoaglin","year":"1978","unstructured":"Hoaglin DC, Welsch RE (1978) The hat matrix in regression and anova. Am Stat 32(1):17\u201322","journal-title":"Am Stat"},{"key":"9518_CR17","unstructured":"Hosmer DW Jr, Lemeshow S (1999) Applied survival analysis: regression modelling of time to event data"},{"issue":"3","key":"9518_CR18","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1197\/jamia.M1733","volume":"12","author":"G Hripcsak","year":"2005","unstructured":"Hripcsak G, Rothschild AS (2005) Agreement, the f-measure, and reliability in information retrieval. J Am Med Inform Assoc 12(3):296\u2013298","journal-title":"J Am Med Inform Assoc"},{"issue":"3","key":"9518_CR19","doi-asserted-by":"crossref","first-page":"299","DOI":"10.1080\/10618600.1996.10474713","volume":"5","author":"R Ihaka","year":"1996","unstructured":"Ihaka R, Gentleman R (1996) R: a language for data analysis and graphics. J Comput Graph Stat 5(3):299\u2013314","journal-title":"J Comput Graph Stat"},{"key":"9518_CR20","doi-asserted-by":"crossref","unstructured":"Kabinna S, Shang W, Bezemer C-P, Hassan AE (2016a) Examining the stability of logging statements. In: SANER 2016: Proceedings of IEEE international conference on the software analysis, evolution and re-engineering. IEEE","DOI":"10.1109\/SANER.2016.29"},{"key":"9518_CR21","doi-asserted-by":"crossref","unstructured":"Kabinna S, Shang W, Bezemer C-P, Hassan AE (2016b) Logging library migrations: a case study for the apache software foundation projects. Mining Software Repositories. page To appear","DOI":"10.1145\/2901739.2901769"},{"issue":"11","key":"9518_CR22","doi-asserted-by":"crossref","first-page":"1073","DOI":"10.1016\/j.infsof.2007.02.015","volume":"49","author":"VB Kampenes","year":"2007","unstructured":"Kampenes VB, Dyb\u00e5 T, Hannay JE, Sj\u00f8berg DIK (2007) A systematic review of effect size in software engineering experiments. Inf Softw Technol 49(11):1073\u20131086","journal-title":"Inf Softw Technol"},{"key":"9518_CR23","unstructured":"Kendall MG (1948) Rank correlation methods. Oxford, England: Griffin Rank correlation methods. http:\/\/psycnet.apa.org\/psycinfo\/1948-15040-000"},{"key":"9518_CR24","unstructured":"Koru AG, Zhang D, Liu H (2007) Modeling the effect of size on defect proneness for open-source software. In: Proceedings of the third international workshop on predictor models in software engineering. IEEE Computer Society, p 10"},{"key":"9518_CR25","doi-asserted-by":"crossref","unstructured":"Li H, Shang W, Hassan AE (2016a) Which log level should developers choose for a new logging statement? Empir Softw Eng. page To appear","DOI":"10.1007\/s10664-016-9456-2"},{"key":"9518_CR26","doi-asserted-by":"crossref","unstructured":"Li H, Shang W, Zou Y, Hassan AE (2016b) Towards just-in-time suggestions for log changes. Empir Softw Eng. page To appear","DOI":"10.1007\/s10664-016-9467-z"},{"key":"9518_CR27","unstructured":"Lou J-G, Fu Q, Yang S, Xu Y, Li J (2010) Mining invariants from console logs for system problem detection. In: Proceedings of the 2010 USENIX conference on USENIX annual technical conference, USENIXATC\u201910, Berkeley, CA, USA. USENIX Association, p 24"},{"key":"9518_CR28","doi-asserted-by":"crossref","unstructured":"Malik H, Hemmati H, Hassan AE (2013) Automatic detection of performance deviations in the load testing of large scale systems. In: Proceedings of (ICSE) 2013, 35th international conference on software engineering, pp 1012\u20131021","DOI":"10.1109\/ICSE.2013.6606651"},{"issue":"5","key":"9518_CR29","doi-asserted-by":"publisher","first-page":"2146","DOI":"10.1007\/s10664-015-9381-9","volume":"21","author":"S McIntosh","year":"2016","unstructured":"McIntosh S, Kamei Y, Adams B, Hassan AE (2016) An empirical study of the impact of modern code review practices on software quality. Empirical Softw Eng 21(5):2146\u20132189. doi: 10.1007\/s10664-015-9381-9","journal-title":"Empirical Softw Eng"},{"issue":"1","key":"9518_CR30","doi-asserted-by":"crossref","first-page":"14","DOI":"10.11592\/bit.121102","volume":"1","author":"M Mednis","year":"2012","unstructured":"Mednis M, Aurich MK (2012) Application of string similarity ratio and edit distance in automatic metabolite reconciliation comparing reconstructions and models. Biosyst Info Technol 1(1):14\u201318","journal-title":"Biosyst Info Technol"},{"key":"9518_CR31","unstructured":"Miller RG Jr (2011) Survival analysis, vol 66. Wiley"},{"key":"9518_CR32","doi-asserted-by":"crossref","unstructured":"Pecchia A, Cinque M, Carrozza G, Cotroneo D (2015) Industry practices and event logging: assessment of a critical software development process. In: Proceedings of the 37th international conference on software engineering, vol 2. IEEE Press, pp 169\u2013178","DOI":"10.1109\/ICSE.2015.145"},{"issue":"16","key":"9518_CR33","doi-asserted-by":"crossref","first-page":"3230","DOI":"10.1200\/JCO.2004.02.080","volume":"22","author":"H Ren","year":"2004","unstructured":"Ren H, Tang X, Lee JJ, Feng L, Everett AD, Hong WK, Khuri FR, Mao L (2004) Expression of hepatoma-derived growth factor is a strong prognostic predictor for patients with early-stage non\u2013small-cell lung cancer. J Clin Oncol 22(16):3230\u20133237","journal-title":"J Clin Oncol"},{"key":"9518_CR34","volume-title":"Information retrieval","author":"CJ Van Rijsbergen","year":"1979","unstructured":"Van Rijsbergen CJ (1979) Information retrieval, 2nd edn. Butterworth-Heinemann, Newton, MA, USA. ISBN 0408709294","edition":"2nd edn."},{"key":"9518_CR35","unstructured":"Serfling RJ (2009) Approximation theorems of mathematical statistics, vol 162. Wiley"},{"key":"9518_CR36","doi-asserted-by":"crossref","unstructured":"Shang W (2012) Bridging the divide between software developers and operators using logs. In: Proceedings of the 34th international conference on software engineering. IEEE, pp 1583\u20131586","DOI":"10.1109\/ICSE.2012.6227031"},{"issue":"1","key":"9518_CR37","first-page":"3","volume":"26","author":"W Shang","year":"2014","unstructured":"Shang W, Jiang ZM, Adams B, Hassan AE, Godfrey MW, Nasser M, Flora P (2014a) An exploratory study of the evolution of communicated information about the execution of large software systems. Journal of Software: Evolution and Process 26(1):3\u201326","journal-title":"Journal of Software: Evolution and Process"},{"key":"9518_CR38","doi-asserted-by":"crossref","unstructured":"Shang W, Nagappan M, Hassan AE, Jiang ZM (2014b) Understanding log lines using development knowledge. In: Proceedings of ICSME 2014, the international conference on software maintenance and evolution. IEEE, pp 21\u201330","DOI":"10.1109\/ICSME.2014.24"},{"issue":"1","key":"9518_CR39","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s10664-013-9274-8","volume":"20","author":"W Shang","year":"2015","unstructured":"Shang W, Nagappan M, Hassan AE (2015) Studying the relationship between logging characteristics and the code quality of platform software. Empir Softw Eng 20(1):1\u201327","journal-title":"Empir Softw Eng"},{"issue":"1","key":"9518_CR40","doi-asserted-by":"crossref","first-page":"307","DOI":"10.1186\/1471-2105-9-307","volume":"9","author":"C Strobl","year":"2008","unstructured":"Strobl C, Boulesteix A-L, Kneib T, Augustin T, Zeileis A (2008) Conditional variable importance for random forests. BMC Bioinforma 9(1):307","journal-title":"BMC Bioinforma"},{"issue":"2","key":"9518_CR41","doi-asserted-by":"crossref","first-page":"176","DOI":"10.1109\/TSE.2014.2361131","volume":"41","author":"M Syer","year":"2015","unstructured":"Syer M, Nagappan M, Adams B, Hassan AE (2015) Replicating and re-evaluating the theory of relative defect-proneness. IEEE Trans Softw Eng 41(2):176\u2013197","journal-title":"IEEE Trans Softw Eng"},{"key":"9518_CR42","unstructured":"Tan J, Pan X, Kavulya S, Gandhi R, Narasimhan P (2008) Salsa: Analyzing logs as state machines. In: WASL\u201908: Proceedings of the 1st USENIX conference on analysis of system logs. USENIX Association, p 6"},{"key":"9518_CR43","unstructured":"Tantithamthavorn C, McIntosh S, Hassan AE, Matsumoto K (2015) An empirical comparison of model validation techniques for defect prediction model. http:\/\/sailhome.cs.queensu.ca\/replication\/kla\/model-validation.pdf . Under Review at Transactions on Software Engineering (TSE)"},{"issue":"1","key":"9518_CR44","doi-asserted-by":"crossref","first-page":"147","DOI":"10.1093\/biomet\/77.1.147","volume":"77","author":"TM Therneau","year":"1990","unstructured":"Therneau TM, Grambsch PM, Fleming TR (1990) Martingale-based residuals for survival models. Biometrika 77(1):147\u2013160","journal-title":"Biometrika"},{"key":"9518_CR45","unstructured":"Log4j Last visited March\u201916. http:\/\/logging.apache.org\/log4j\/2.x\/"},{"key":"9518_CR46","unstructured":"Xpolog. http:\/\/www.xpolog.com\/"},{"key":"9518_CR47","doi-asserted-by":"crossref","unstructured":"Xu W, Huang L, Fox A, Patterson D, Jordan MI (2009) Detecting large-scale system problems by mining console logs. In: Proceedings of the ACM SOPS 2009, 22nd symposium on operating systems principle, pp 117\u2013132","DOI":"10.1145\/1629575.1629587"},{"key":"9518_CR48","doi-asserted-by":"crossref","unstructured":"Xu X, Weber I, Bass L, Zhu L, Wada H, Teng F (2013) Detecting cloud provisioning errors using an annotated process model. In: Proceedings of MW4NG 2013, the 8th workshop on middleware for next generation internet computing. ACM, p 5","DOI":"10.1145\/2541608.2541613"},{"key":"9518_CR49","unstructured":"Yuan D, Park S, Huang P, Liu Y, Lee MM, Tang X, Zhou Y, Savage S (2012) Be conservative: enhancing failure diagnosis with proactive logging. In: OSDI 2012, USENIX Symposium on operating systems design and implementation, pp 293\u2013306"},{"key":"9518_CR50","doi-asserted-by":"crossref","unstructured":"Yuan D, Zheng J, Park S, Zhou Y, Savage S (2011) Improving software diagnosability via log enhancement. In: Proceedings of ASPLOS 2011, the 16th conference on architectural support for programming languages and operating systems, pp 3\u201314","DOI":"10.1145\/1950365.1950369"},{"key":"9518_CR51","doi-asserted-by":"crossref","unstructured":"Yuan D, Park S, Zhou Y (2012) Characterizing logging practices in open-source software. In: Proceedings of ICSE 2012, the 34th international conference on software engineering. IEEE Press, pp 102\u2013112","DOI":"10.1109\/ICSE.2012.6227202"},{"issue":"2","key":"9518_CR52","doi-asserted-by":"crossref","first-page":"293","DOI":"10.1109\/TSE.2008.90","volume":"35","author":"D Zhang","year":"2009","unstructured":"Zhang D, El Emam K, Liu H (2009) An investigation into the functional form of the size-defect relationship for software modules. IEEE Trans Softw Eng 35(2):293\u2013304","journal-title":"IEEE Trans Softw Eng"},{"key":"9518_CR53","doi-asserted-by":"crossref","unstructured":"Zhu J, He P, Fu Q, Zhang H, Lyu MR, Zhang D (2015) Learning to log: helping developers make informed logging decisions. In: Proceedings of ICSE 2015, the 37th international conference on software engineering, vol 1. IEEE Press, Piscataway, NJ, USA, pp 415\u2013425","DOI":"10.1109\/ICSE.2015.60"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-017-9518-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-017-9518-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-017-9518-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,24]],"date-time":"2024-06-24T20:07:26Z","timestamp":1719259646000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-017-9518-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,6,15]]},"references-count":53,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2018,2]]}},"alternative-id":["9518"],"URL":"https:\/\/doi.org\/10.1007\/s10664-017-9518-0","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,6,15]]}}}