{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,30]],"date-time":"2026-01-30T05:44:00Z","timestamp":1769751840040,"version":"3.49.0"},"publisher-location":"Cham","reference-count":38,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783031274800","type":"print"},{"value":"9783031274817","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-27481-7_33","type":"book-chapter","created":{"date-parts":[[2023,3,2]],"date-time":"2023-03-02T14:03:10Z","timestamp":1677765790000},"page":"584-597","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Shifting Left for\u00a0Early Detection of\u00a0Machine-Learning Bugs"],"prefix":"10.1007","author":[{"given":"Ben","family":"Liblit","sequence":"first","affiliation":[]},{"given":"Linghui","family":"Luo","sequence":"additional","affiliation":[]},{"given":"Alejandro","family":"Molina","sequence":"additional","affiliation":[]},{"given":"Rajdeep","family":"Mukherjee","sequence":"additional","affiliation":[]},{"given":"Zachary","family":"Patterson","sequence":"additional","affiliation":[]},{"given":"Goran","family":"Piskachev","sequence":"additional","affiliation":[]},{"given":"Martin","family":"Sch\u00e4f","sequence":"additional","affiliation":[]},{"given":"Omer","family":"Tripp","sequence":"additional","affiliation":[]},{"given":"Willem","family":"Visser","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,3,3]]},"reference":[{"key":"33_CR1","unstructured":"Abadi, M., et al.: TensorFlow: large-scale machine learning on heterogeneous systems (2015). https:\/\/www.tensorflow.org\/"},{"key":"33_CR2","unstructured":"Apache: Apache MXNet (2022). https:\/\/mxnet.apache.org\/versions\/1.9.1\/"},{"issue":"2","key":"33_CR3","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1145\/1646353.1646374","volume":"53","author":"A Bessey","year":"2010","unstructured":"Bessey, A., et al.: A few billion lines of code later: using static analysis to find bugs in the real world. Commun. ACM 53(2), 66\u201375 (2010). https:\/\/doi.org\/10.1145\/1646353.1646374. ISSN 0001-0782","journal-title":"Commun. ACM"},{"key":"33_CR4","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511804441","volume-title":"Convex Optimization","author":"S Boyd","year":"2004","unstructured":"Boyd, S., Boyd, S.P., Vandenberghe, L.: Convex Optimization. Cambridge University Press, Cambridge (2004)"},{"key":"33_CR5","unstructured":"Chollet, F., et al.: Keras (2015). https:\/\/keras.io"},{"key":"33_CR6","doi-asserted-by":"publisher","unstructured":"Dilhara, M., Ketkar, A., Dig, D.: Understanding software-2.0: a study of machine learning library usage and evolution. ACM Trans. Softw. Eng. Methodol. 30(4) (2021). https:\/\/doi.org\/10.1145\/3453478. ISSN 1049-331X","DOI":"10.1145\/3453478"},{"issue":"8","key":"33_CR7","doi-asserted-by":"publisher","first-page":"62","DOI":"10.1145\/3338112","volume":"62","author":"D Distefano","year":"2019","unstructured":"Distefano, D., F\u00e4hndrich, M., Logozzo, F., O\u2019Hearn, P.W.: Scaling static analyses at Facebook. Commun. ACM 62(8), 62\u201370 (2019). https:\/\/doi.org\/10.1145\/3338112. ISSN 0001-0782","journal-title":"Commun. ACM"},{"key":"33_CR8","doi-asserted-by":"publisher","unstructured":"Dolby, J., Shinnar, A., Allain, A., Reinen, J.: Ariadne: analysis for machine learning programs. In: Proceedings of the 2nd ACM SIGPLAN International Workshop on Machine Learning and Programming Languages, MAPL 2018, pp. 1\u201310. Association for Computing Machinery, New York (2018). https:\/\/doi.org\/10.1145\/3211346.3211349. ISBN 9781450358347","DOI":"10.1145\/3211346.3211349"},{"key":"33_CR9","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"406","DOI":"10.1007\/3-540-47910-4_21","volume-title":"ECOOP\u2019 93 \u2014 Object-Oriented Programming","author":"E Gamma","year":"1993","unstructured":"Gamma, E., Helm, R., Johnson, R., Vlissides, J.: Design patterns: abstraction and reuse of object-oriented design. In: Nierstrasz, O.M. (ed.) ECOOP 1993. LNCS, vol. 707, pp. 406\u2013431. Springer, Heidelberg (1993). https:\/\/doi.org\/10.1007\/3-540-47910-4_21"},{"key":"33_CR10","doi-asserted-by":"publisher","unstructured":"Gehr, T., Mirman, M., Drachsler-Cohen, D., Tsankov, P., Chaudhuri, S., Vechev, M.: AI2: safety and robustness certification of neural networks with abstract interpretation. In: 2018 IEEE Symposium on Security and Privacy (SP), pp. 3\u201318 (2018). https:\/\/doi.org\/10.1109\/SP.2018.00058","DOI":"10.1109\/SP.2018.00058"},{"key":"33_CR11","doi-asserted-by":"publisher","unstructured":"Grotov, K., Titov, S., Sotnikov, V., Golubev, Y., Bryksin, T.: A large-scale comparison of Python code in Jupyter notebooks and scripts. In: Proceedings of the 19th International Conference on Mining Software Repositories, MSR 2022, pp. 353\u2013364. Association for Computing Machinery, New York (2022). https:\/\/doi.org\/10.1145\/3524842.3528447. ISBN 9781450393034","DOI":"10.1145\/3524842.3528447"},{"issue":"1","key":"33_CR12","doi-asserted-by":"publisher","first-page":"59","DOI":"10.1177\/1525822X05279903","volume":"18","author":"G Guest","year":"2006","unstructured":"Guest, G., Bunce, A., Johnson, L.: How many interviews are enough? An experiment with data saturation and variability. Field Methods 18(1), 59\u201382 (2006)","journal-title":"Field Methods"},{"key":"33_CR13","doi-asserted-by":"publisher","unstructured":"Humbatova, N., Jahangirova, G., Bavota, G., Riccio, V., Stocco, A., Tonella, P.: Taxonomy of real faults in deep learning systems. In: Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering, ICSE 2020, pp. 1110\u20131121. Association for Computing Machinery, New York (2020). https:\/\/doi.org\/10.1145\/3377811.3380395. ISBN 9781450371216","DOI":"10.1145\/3377811.3380395"},{"key":"33_CR14","unstructured":"Ioffe, S., Szegedy, C.: Batch normalization: accelerating deep network training by reducing internal covariate shift. In: International Conference on Machine Learning, pp. 448\u2013456. PMLR (2015)"},{"key":"33_CR15","unstructured":"Kluyver, T., et al.: Jupyter notebooks - a publishing format for reproducible computational workflows. In: Loizides, F., Scmidt, B. (eds.) Positioning and Power in Academic Publishing: Players, Agents and Agendas, pp. 87\u201390. IOS Press (2016). https:\/\/eprints.soton.ac.uk\/403913\/"},{"key":"33_CR16","doi-asserted-by":"publisher","unstructured":"Lagouvardos, S., Dolby, J., Grech, N., Antoniadis, A., Smaragdakis, Y.: Static analysis of shape in TensorFlow programs. In: Hirschfeld, R., Pape, T. (eds.) 34th European Conference on Object-Oriented Programming (ECOOP 2020). Leibniz International Proceedings in Informatics (LIPIcs), vol. 166, pp. 15:1\u201315:29. Schloss Dagstuhl-Leibniz-Zentrum f\u00fcr Informatik, Dagstuhl (2020). https:\/\/doi.org\/10.4230\/LIPIcs.ECOOP.2020.15, https:\/\/drops.dagstuhl.de\/opus\/volltexte\/2020\/13172. ISBN 978-3-95977-154-2, ISSN 1868-8969","DOI":"10.4230\/LIPIcs.ECOOP.2020.15"},{"key":"33_CR17","unstructured":"LeCun, Y., Touresky, D., Hinton, G., Sejnowski, T.: A theoretical framework for back-propagation. In: Proceedings of the 1988 Connectionist Models Summer School, vol. 1, pp. 21\u201328 (1988)"},{"key":"33_CR18","doi-asserted-by":"publisher","unstructured":"Liu, C., et al.: Detecting TensorFlow program bugs in real-world industrial environment. In: 2021 36th IEEE\/ACM International Conference on Automated Software Engineering (ASE), pp. 55\u201366 (2021). https:\/\/doi.org\/10.1109\/ASE51524.2021.9678891","DOI":"10.1109\/ASE51524.2021.9678891"},{"key":"33_CR19","doi-asserted-by":"crossref","unstructured":"Madhyastha, P., Jain, R.: On model stability as a function of random seed. arXiv preprint arXiv:1909.10447 (2019)","DOI":"10.18653\/v1\/K19-1087"},{"key":"33_CR20","unstructured":"Microsoft: Pyright: Static type checker for Python (2022). https:\/\/github.com\/microsoft\/pyright"},{"key":"33_CR21","doi-asserted-by":"publisher","unstructured":"Mukherjee, R., Tripp, O., Liblit, B., Wilson, M.: Static analysis for AWS best practices in Python code. In: Ali, K., Vitek, J. (eds.) 36th European Conference on Object-Oriented Programming, ECOOP 2022, 6\u201310 June 2022, Berlin, Germany. LIPIcs, vol. 222, pp. 14:1\u201314:28. Schloss Dagstuhl - Leibniz-Zentrum f\u00fcr Informatik (2022), https:\/\/doi.org\/10.4230\/LIPIcs.ECOOP.2022.14","DOI":"10.4230\/LIPIcs.ECOOP.2022.14"},{"key":"33_CR22","unstructured":"Paszke, A., et al.: PyTorch: an imperative style, high-performance deep learning library. Adv. Neural Inf. Process. Syst. 32 (2019)"},{"key":"33_CR23","doi-asserted-by":"publisher","unstructured":"Pimentel, J.A.F., Murta, L., Braganholo, V., Freire, J.: A large-scale study about quality and reproducibility of Jupyter notebooks. In: Proceedings of the 16th International Conference on Mining Software Repositories, MSR 2019, pp. 507\u2013517. IEEE Press (2019). https:\/\/doi.org\/10.1109\/MSR.2019.00077","DOI":"10.1109\/MSR.2019.00077"},{"key":"33_CR24","unstructured":"Python Software Foundation: The Python standard library: typing\u2014support for type hints: typing.Union (2022). https:\/\/docs.python.org\/3\/library\/typing.html#typing.Union"},{"key":"33_CR25","unstructured":"Python Software Foundation: The Python standard library: typing\u2014support for type hints: The Any type (2022). https:\/\/docs.python.org\/3\/library\/typing.html#the-any-type"},{"key":"33_CR26","doi-asserted-by":"publisher","unstructured":"Quaranta, L.: Assessing the quality of computational notebooks for a frictionless transition from exploration to production. In: Proceedings of the ACM\/IEEE 44th International Conference on Software Engineering: Companion Proceedings, ICSE 2022, pp. 256\u2013260. Association for Computing Machinery, New York (2022). https:\/\/doi.org\/10.1145\/3510454.3517055. ISBN 9781450392235","DOI":"10.1145\/3510454.3517055"},{"key":"33_CR27","doi-asserted-by":"publisher","unstructured":"Quaranta, L., Calefato, F., Lanubile, F.: Pynblint: a static analyzer for Python Jupyter notebooks. In: 2022 IEEE\/ACM 1st International Conference on AI Engineering - Software Engineering for AI (CAIN), pp. 48\u201349 (2022). https:\/\/doi.org\/10.1145\/3522664.3528612","DOI":"10.1145\/3522664.3528612"},{"key":"33_CR28","doi-asserted-by":"crossref","unstructured":"Rasley, J., Rajbhandari, S., Ruwase, O., He, Y.: DeepSpeed: system optimizations enable training deep learning models with over 100 billion parameters. In: Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, pp. 3505\u20133506 (2020)","DOI":"10.1145\/3394486.3406703"},{"key":"33_CR29","unstructured":"Research, I.: WALA: The T. J. Watson libraries for analysis (2022). https:\/\/github.com\/wala\/WALA"},{"key":"33_CR30","unstructured":"Ruder, S.: An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747 (2016)"},{"issue":"1","key":"33_CR31","first-page":"1929","volume":"15","author":"N Srivastava","year":"2014","unstructured":"Srivastava, N., Hinton, G., Krizhevsky, A., Sutskever, I., Salakhutdinov, R.: Dropout: a simple way to prevent neural networks from overfitting. J. Mach. Learn. Res. 15(1), 1929\u20131958 (2014)","journal-title":"J. Mach. Learn. Res."},{"key":"33_CR32","doi-asserted-by":"publisher","unstructured":"Suboti\u0107, P., Miliki\u0107, L., Stoji\u0107, M.: A static analysis framework for data science notebooks. In: Proceedings of the 44th International Conference on Software Engineering: Software Engineering in Practice, ICSE-SEIP 2022, pp. 13\u201322. Association for Computing Machinery, New York (2022). https:\/\/doi.org\/10.1145\/3510457.3513032. ISBN 9781450392266","DOI":"10.1145\/3510457.3513032"},{"key":"33_CR33","doi-asserted-by":"publisher","unstructured":"Urban, C.: Static analysis of data science software. In: Chang, B.-Y.E. (ed.) SAS 2019. LNCS, vol. 11822, pp. 17\u201323. Springer, Cham (2019). https:\/\/doi.org\/10.1007\/978-3-030-32304-2_2. ISBN 978-3-030-32304-2","DOI":"10.1007\/978-3-030-32304-2_2"},{"key":"33_CR34","doi-asserted-by":"publisher","unstructured":"Wan, C., Liu, S., Hoffmann, H., Maire, M., Lu, S.: Are machine learning cloud APIs used correctly? In: 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE), pp. 125\u2013137 (2021). https:\/\/doi.org\/10.1109\/ICSE43902.2021.00024","DOI":"10.1109\/ICSE43902.2021.00024"},{"issue":"9","key":"33_CR35","doi-asserted-by":"publisher","first-page":"1857","DOI":"10.1109\/TSE.2019.2937083","volume":"47","author":"Z Wan","year":"2021","unstructured":"Wan, Z., Xia, X., Lo, D., Murphy, G.C.: How does machine learning change software development practices? IEEE Trans. Software Eng. 47(9), 1857\u20131871 (2021). https:\/\/doi.org\/10.1109\/TSE.2019.2937083","journal-title":"IEEE Trans. Software Eng."},{"key":"33_CR36","doi-asserted-by":"crossref","unstructured":"Wang, J., Kuo, T.y., Li, L., Zeller, A.: Restoring reproducibility of Jupyter notebooks. In: 2020 IEEE\/ACM 42nd International Conference on Software Engineering: Companion Proceedings (ICSE-Companion), pp. 288\u2013289 (2020)","DOI":"10.1145\/3377812.3390803"},{"key":"33_CR37","doi-asserted-by":"publisher","unstructured":"Wu, D., Shen, B., Chen, Y., Jiang, H., Qiao, L.: Tensfa: detecting and repairing tensor shape faults in deep learning systems. In: 2021 IEEE 32nd International Symposium on Software Reliability Engineering (ISSRE), pp. 11\u201321 (2021). https:\/\/doi.org\/10.1109\/ISSRE52982.2021.00014","DOI":"10.1109\/ISSRE52982.2021.00014"},{"key":"33_CR38","doi-asserted-by":"publisher","unstructured":"Zhang, Y., Ren, L., Chen, L., Xiong, Y., Cheung, S.C., Xie, T.: Detecting numerical bugs in neural network architectures. In: Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC\/FSE 2020, pp. 826\u2013837. Association for Computing Machinery, New York (2020). https:\/\/doi.org\/10.1145\/3368089.3409720. ISBN 9781450370431","DOI":"10.1145\/3368089.3409720"}],"container-title":["Lecture Notes in Computer Science","Formal Methods"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-27481-7_33","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,3,2]],"date-time":"2023-03-02T14:11:43Z","timestamp":1677766303000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-27481-7_33"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031274800","9783031274817"],"references-count":38,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-27481-7_33","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"3 March 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"FM","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Symposium on Formal Methods","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"L\u00fcbeck","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Germany","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 March 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 March 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"fm2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/fm2023.isp.uni-luebeck.de\/wordpress\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"95","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"26","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"27% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The proceedings also include 7 short industry papers","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}