{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T02:37:15Z","timestamp":1784169435032,"version":"3.55.0"},"reference-count":58,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2023,11,29]],"date-time":"2023-11-29T00:00:00Z","timestamp":1701216000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,11,29]],"date-time":"2023-11-29T00:00:00Z","timestamp":1701216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"name":"National Science and Engineering Research Council of Canada","award":["CRDPJ 537462-18"],"award-info":[{"award-number":["CRDPJ 537462-18"]}]},{"DOI":"10.13039\/100007631","name":"Canadian Institute for Advanced Research","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100007631","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100020951","name":"Fonds de Recherche du Qu\u00e9bec","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100020951","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100015773","name":"Consortium de Recherche et d\u2019innovation en A\u00e9rospatiale au Qu\u00e9bec","doi-asserted-by":"publisher","award":["CRDPJ 537462-18"],"award-info":[{"award-number":["CRDPJ 537462-18"]}],"id":[{"id":"10.13039\/100015773","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2024,1]]},"DOI":"10.1007\/s10664-023-10389-6","type":"journal-article","created":{"date-parts":[[2023,11,29]],"date-time":"2023-11-29T11:02:21Z","timestamp":1701255741000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":38,"title":["Silent bugs in deep learning frameworks: an empirical study of Keras and TensorFlow"],"prefix":"10.1007","volume":"29","author":[{"given":"Florian","family":"Tambon","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0440-6839","authenticated-orcid":false,"given":"Amin","family":"Nikanjam","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Le","family":"An","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Foutse","family":"Khomh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Giuliano","family":"Antoniol","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,11,29]]},"reference":[{"key":"10389_CR1","unstructured":"(2020) Keras releases. https:\/\/github.com\/keras-team\/keras\/releases\/tag\/2.4.0"},{"key":"10389_CR2","unstructured":"(2020) TenforFlow implementation. https:\/\/github.com\/tensorflow\/tensorflow\/tree\/master\/tensorflow\/python\/keras"},{"key":"10389_CR3","unstructured":"(2020) tf.keras API. https:\/\/www.tensorflow.org\/api_docs\/python\/tf\/keras"},{"key":"10389_CR4","unstructured":"(2021) DL Frameworks in 2021. https:\/\/towardsdatascience.com\/top-5-deep-learning-frameworks-to-watch-in-2021-and-why-tensorflow-98d8d6667351"},{"key":"10389_CR5","unstructured":"(2022) Keras. https:\/\/keras.io\/"},{"key":"10389_CR6","unstructured":"(2022) Pytorch. https:\/\/pytorch.org\/"},{"key":"10389_CR7","unstructured":"(2022) Replication Package. https:\/\/github.com\/amin-nikanjam\/SilentBugsInTensorFlowKeras"},{"key":"10389_CR8","unstructured":"(2022) Tensorflow. https:\/\/www.tensorflow.org\/"},{"key":"10389_CR9","unstructured":"(2022) TensorFlow repository. https:\/\/github.com\/tensorflow\/tensorflow"},{"key":"10389_CR10","unstructured":"Abadi M, Barham P, Chen J, Chen Z, Davis A, Dean J, Devin M, Ghemawat S, Irving G, Isard M, et\u00a0al (2016) Tensorflow: A system for large-scale machine learning. In: 12th $$\\{$$USENIX$$\\}$$ symposium on operating systems design and implementation ($$\\{$$OSDI$$\\}$$ 16), p 265\u2013283"},{"key":"10389_CR11","volume-title":"Practical statistics for medical research","author":"DG Altman","year":"1991","unstructured":"Altman DG (1991) Practical statistics for medical research. Chapman and Hall, London"},{"key":"10389_CR12","doi-asserted-by":"publisher","unstructured":"Chen J, Liang Y, Shen Q, Jiang J, Li S (2023) Toward understanding deep learning framework bugs. ACM Trans Softw Eng Methodol. https:\/\/doi.org\/10.1145\/3587155 just Accepted","DOI":"10.1145\/3587155"},{"issue":"1","key":"10389_CR13","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen J (1960) A coefficient of agreement for nominal scales. Educ Psychol Meas 20(1):37\u201346","journal-title":"Educ Psychol Meas"},{"key":"10389_CR14","doi-asserted-by":"publisher","unstructured":"Di\u00a0Franco A, Guo H, Rubio-Gonz\u00e1lez C (2017) A comprehensive study of real-world numerical bug characteristics. In: 2017 32nd IEEE\/ACM international conference on automated software engineering (ASE), p 509\u2013519. https:\/\/doi.org\/10.1109\/ASE.2017.8115662","DOI":"10.1109\/ASE.2017.8115662"},{"key":"10389_CR15","doi-asserted-by":"crossref","unstructured":"Du X, Xiao G, Sui Y (2020) Fault triggers in the tensorflow framework: An experience report. In: 2020 IEEE 31st international symposium on software reliability engineering (ISSRE), IEEE, p 1\u201312","DOI":"10.1109\/ISSRE5003.2020.00010"},{"key":"10389_CR16","doi-asserted-by":"crossref","unstructured":"Du X, Sui Y, Liu Z, Ai J (2022) An empirical study of fault triggers in deep learning frameworks. IEEE Trans Depend Sec Comput","DOI":"10.1109\/TDSC.2022.3152239"},{"key":"10389_CR17","unstructured":"fix-example (2020) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/commit\/15f6c30d7977c92ba452eb5c1873b8c9f0968a5f"},{"key":"10389_CR18","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"JL Fleiss","year":"1971","unstructured":"Fleiss JL (1971) Measuring nominal scale agreement among many raters. Psychol Bull 76:378\u2013382","journal-title":"Psychol Bull"},{"key":"10389_CR19","unstructured":"githubREST (2022) Github REST API. https:\/\/docs.github.com\/en\/rest"},{"key":"10389_CR20","unstructured":"GitSearchAPI (2022) Github search api. https:\/\/docs.github.com\/en\/rest\/reference\/search"},{"key":"10389_CR21","unstructured":"googleForm (2022) Google forms. https:\/\/www.google.ca\/forms\/about\/"},{"issue":"3","key":"10389_CR22","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1109\/TSE.2013.59","volume":"40","author":"A Groce","year":"2013","unstructured":"Groce A, Kulesza T, Zhang C, Shamasunder S, Burnett M, Wong WK, Stumpf S, Das S, Shinsel A, Bice F et al (2013) You are the only possible oracle: Effective test selection for end users of interactive machine learning systems. IEEE Trans Soft Eng 40(3):307\u2013323","journal-title":"IEEE Trans Soft Eng"},{"key":"10389_CR23","doi-asserted-by":"crossref","unstructured":"Humbatova N, Jahangirova G, Bavota G, Riccio V, Stocco A, Tonella P (2020) Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd international conference on software engineering, p 1110\u20131121","DOI":"10.1145\/3377811.3380395"},{"key":"10389_CR24","doi-asserted-by":"crossref","unstructured":"Islam MJ, Nguyen G, Pan R, Rajan H (2019) A comprehensive study on deep learning bug characteristics. In: Proceedings of the 2019 27th ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, p 510\u2013520","DOI":"10.1145\/3338906.3338955"},{"key":"10389_CR25","unstructured":"issue1 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/42459"},{"key":"10389_CR26","unstructured":"issue2 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/32476"},{"key":"10389_CR27","unstructured":"issue3 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/32286"},{"key":"10389_CR28","unstructured":"issue4 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/31324"},{"key":"10389_CR29","unstructured":"issue5 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/32420"},{"key":"10389_CR30","unstructured":"issue6 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/30486"},{"key":"10389_CR31","unstructured":"issue7 (2022) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/38596"},{"key":"10389_CR32","unstructured":"issue8 (2020) TensorFlow\/Keras Issue. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/38197"},{"key":"10389_CR33","unstructured":"jax (2022) JAX. https:\/\/jax.readthedocs.io\/en\/latest\/"},{"key":"10389_CR34","doi-asserted-by":"crossref","unstructured":"Jia L, Zhong H, Wang X, Huang L, Lu X (2020) An empirical study on bugs inside tensorflow. In: International conference on database systems for advanced applications, Springer, p 604\u2013620","DOI":"10.1007\/978-3-030-59410-7_40"},{"key":"10389_CR35","doi-asserted-by":"publisher","unstructured":"Jia L, Zhong H, Huang L (2021a) The unit test quality of deep learning libraries: A mutation analysis. In: 2021 IEEE international conference on software maintenance and evolution (ICSME), p 47\u201357. https:\/\/doi.org\/10.1109\/ICSME52107.2021.00011","DOI":"10.1109\/ICSME52107.2021.00011"},{"key":"10389_CR36","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.110935","volume":"177","author":"L Jia","year":"2021","unstructured":"Jia L, Zhong H, Wang X, Huang L, Lu X (2021) The symptoms, causes, and repairs of bugs inside a deep learning library. J Syst Soft 177:110935","journal-title":"J Syst Soft"},{"key":"10389_CR37","doi-asserted-by":"publisher","unstructured":"Jia L, Zhong H, Wang X, Huang L, Li Z (2022) How do injected bugs affect deep learning? In: 2022 IEEE international conference on software analysis, evolution and reengineering (SANER), p 793\u2013804. https:\/\/doi.org\/10.1109\/SANER53432.2022.00097","DOI":"10.1109\/SANER53432.2022.00097"},{"key":"10389_CR38","doi-asserted-by":"publisher","unstructured":"Kendall MG (1938) A new measure of rank correlation. Biometrika 30(1\u20132):81\u201393. https:\/\/doi.org\/10.1093\/biomet\/30.1-2.81, https:\/\/academic.oup.com\/biomet\/article-pdf\/30\/1-2\/81\/423380\/30-1-2-81.pdf","DOI":"10.1093\/biomet\/30.1-2.81"},{"key":"10389_CR39","doi-asserted-by":"publisher","unstructured":"Kouwe EVD, Giuffrida C, Tanenbaum AS (2014) On the soundness of silence: Investigating silent failures using fault injection experiments. In: 2014 Tenth European dependable computing conference, p 118\u2013129. https:\/\/doi.org\/10.1109\/EDCC.2014.16","DOI":"10.1109\/EDCC.2014.16"},{"key":"10389_CR40","doi-asserted-by":"publisher","unstructured":"Le V, Afshari M, Su Z (2014) Compiler validation via equivalence modulo inputs. In: Proceedings of the 35th ACM SIGPLAN conference on programming language design and implementation, association for computing machinery, New York, NY, USA, PLDI \u201914, p 216\u2013226. https:\/\/doi.org\/10.1145\/2594291.2594334","DOI":"10.1145\/2594291.2594334"},{"key":"10389_CR41","doi-asserted-by":"publisher","unstructured":"Li M, Cao J, Tian Y, Li TO, Wen M, Cheung SC (2023) Comet: Coverage-guided model generation for deep learning library testing. ACM Trans Softw Eng Methodol. https:\/\/doi.org\/10.1145\/3583566 just Accepted","DOI":"10.1145\/3583566"},{"key":"10389_CR42","doi-asserted-by":"publisher","unstructured":"Long F, Rinard M (2016) Automatic patch generation by learning correct code. In: Proceedings of the 43rd annual ACM SIGPLAN-SIGACT symposium on principles of programming languages, association for computing machinery, New York, NY, USA, POPL \u201916, p 298\u2013312. https:\/\/doi.org\/10.1145\/2837614.2837617","DOI":"10.1145\/2837614.2837617"},{"key":"10389_CR43","unstructured":"MLlib-Spark (2022) MLlib-Spark. https:\/\/spark.apache.org\/mllib\/"},{"key":"10389_CR44","doi-asserted-by":"publisher","first-page":"77","DOI":"10.1007\/s10462-018-09679-z","volume":"52","author":"G Nguyen","year":"2019","unstructured":"Nguyen G, Dlugolinsky S, Bob\u00e1k M, Tran V, L\u00f3pez Garc\u00eda \u00c1, Heredia I, Mal\u00edk P, Hluch\u00fd L (2019) Machine learning and deep learning frameworks and libraries for large-scale data mining: a survey. Artif Intell Rev 52:77\u2013124","journal-title":"Artif Intell Rev"},{"key":"10389_CR45","doi-asserted-by":"crossref","unstructured":"Nikanjam A, Morovati MM, Khomh F, Ben\u00a0Braiek H (2021) Faults in deep reinforcement learning programs: a taxonomy and a detection approach. Auto Soft Eng 29","DOI":"10.1007\/s10515-021-00313-x"},{"key":"10389_CR46","volume-title":"Questionnaire design, interviewing and attitude measurement","author":"AN Oppenheim","year":"2000","unstructured":"Oppenheim AN (2000) Questionnaire design, interviewing and attitude measurement. Bloomsbury Publishing"},{"key":"10389_CR47","doi-asserted-by":"publisher","unstructured":"Papadakis M, Shin D, Yoo S, Bae DH (2018) Are mutation scores correlated with real fault detection? a large scale empirical study on the relationship between mutants and real faults. In: Proceedings of the 40th international conference on software engineering, association for computing machinery, New York, NY, USA, ICSE \u201918, p 537\u2013548. https:\/\/doi.org\/10.1145\/3180155.3180183","DOI":"10.1145\/3180155.3180183"},{"key":"10389_CR48","doi-asserted-by":"publisher","unstructured":"Pham HV, Lutellier T, Qi W, Tan L (2019) Cradle: Cross-backend validation to detect and localize bugs in deep learning libraries. In: 2019 IEEE\/ACM 41st international conference on software engineering (ICSE), pp 1027\u20131038. https:\/\/doi.org\/10.1109\/ICSE.2019.00107","DOI":"10.1109\/ICSE.2019.00107"},{"key":"10389_CR49","unstructured":"reddit (2022) Reddit. https:\/\/www.reddit.com\/"},{"issue":"4","key":"10389_CR50","doi-asserted-by":"publisher","first-page":"557","DOI":"10.1109\/32.799955","volume":"25","author":"CB Seaman","year":"1999","unstructured":"Seaman CB (1999) Qualitative methods in empirical studies of software engineering. IEEE Transactions on software engineering 25(4):557\u2013572","journal-title":"IEEE Transactions on software engineering"},{"issue":"10","key":"10389_CR51","doi-asserted-by":"publisher","first-page":"849","DOI":"10.1145\/3022671.2984038","volume":"51","author":"C Sun","year":"2016","unstructured":"Sun C, Le V, Su Z (2016) Finding compiler bugs via live code mutation. SIGPLAN Not 51(10):849\u2013863. https:\/\/doi.org\/10.1145\/3022671.2984038","journal-title":"SIGPLAN Not"},{"key":"10389_CR52","doi-asserted-by":"publisher","unstructured":"Vahabzadeh A, Fard AM, Mesbah A (2015) An empirical study of bugs in test code. In: 2015 IEEE International conference on software maintenance and evolution (ICSME), p 101\u2013110. https:\/\/doi.org\/10.1109\/ICSM.2015.7332456","DOI":"10.1109\/ICSM.2015.7332456"},{"key":"10389_CR53","doi-asserted-by":"crossref","unstructured":"Wang W, Poo-Caama\u00f1o G, Wilde E, German DM (2015) What is the gist? understanding the use of public gists on github. In: 2015 IEEE\/ACM 12th Working conference on mining software repositories, IEEE, p 314\u2013323","DOI":"10.1109\/MSR.2015.36"},{"key":"10389_CR54","doi-asserted-by":"crossref","unstructured":"Wang Z, Yan M, Chen J, Liu S, Zhang D (2020) Deep learning library testing via effective model generation. In: Proceedings of the 28th ACM Joint meeting on European software engineering conference and symposium on the foundations of software engineering, p 788\u2013799","DOI":"10.1145\/3368089.3409761"},{"key":"10389_CR55","doi-asserted-by":"publisher","unstructured":"Weimer W, Nguyen T, Le\u00a0Goues C, Forrest S (2009) Automatically finding patches using genetic programming. In: 2009 IEEE 31st international conference on software engineering, p 364\u2013374. https:\/\/doi.org\/10.1109\/ICSE.2009.5070536","DOI":"10.1109\/ICSE.2009.5070536"},{"issue":"1","key":"10389_CR56","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1109\/TSE.2019.2962027","volume":"48","author":"JM Zhang","year":"2022","unstructured":"Zhang JM, Harman M, Ma L, Liu Y (2022) Machine learning testing: Survey, landscapes and horizons. IEEE Trans Soft Eng 48(1):1\u201336. https:\/\/doi.org\/10.1109\/TSE.2019.2962027","journal-title":"IEEE Trans Soft Eng"},{"key":"10389_CR57","doi-asserted-by":"crossref","unstructured":"Zhang Y, Chen Y, Cheung SC, Xiong Y, Zhang L (2018a) An empirical study on tensorflow program bugs. In: Proceedings of the 27th ACM SIGSOFT international symposium on software testing and analysis, p 129\u2013140","DOI":"10.1145\/3213846.3213866"},{"key":"10389_CR58","doi-asserted-by":"crossref","unstructured":"Zhang Y, Chen Y, Cheung SC, Xiong Y, Zhang L (2018b) An empirical study on tensorflow program bugs. In: Proceedings of the 27th ACM SIGSOFT international symposium on software testing and analysis, p 129\u2013140","DOI":"10.1145\/3213846.3213866"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10389-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-023-10389-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-023-10389-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,20]],"date-time":"2024-01-20T03:12:22Z","timestamp":1705720342000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-023-10389-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,29]]},"references-count":58,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2024,1]]}},"alternative-id":["10389"],"URL":"https:\/\/doi.org\/10.1007\/s10664-023-10389-6","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,11,29]]},"assertion":[{"value":"30 August 2023","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 November 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest"}}],"article-number":"10"}}