{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,11]],"date-time":"2026-04-11T02:12:17Z","timestamp":1775873537757,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,7,18]],"date-time":"2020-07-18T00:00:00Z","timestamp":1595030400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,7,18]]},"DOI":"10.1145\/3395363.3404540","type":"proceedings-article","created":{"date-parts":[[2020,7,13]],"date-time":"2020-07-13T21:44:18Z","timestamp":1594676658000},"page":"189-199","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["Detecting and understanding real-world differential performance bugs in machine learning libraries"],"prefix":"10.1145","author":[{"given":"Saeid","family":"Tizpaz-Niari","sequence":"first","affiliation":[{"name":"University of Colorado Boulder, USA"}]},{"given":"Pavol","family":"\u010cern\u00fd","sequence":"additional","affiliation":[{"name":"TU Vienna, Austria"}]},{"given":"Ashutosh","family":"Trivedi","sequence":"additional","affiliation":[{"name":"University of Colorado Boulder, USA"}]}],"member":"320","published-online":{"date-parts":[[2020,7,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"AFL. 2016. American fuzzy lop. http:\/\/lcamtuf.coredump.cx\/afl\/. Online.  AFL. 2016. American fuzzy lop. http:\/\/lcamtuf.coredump.cx\/afl\/. Online."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","volume-title":"Classification and regression trees","author":"Breiman Leo","DOI":"10.1201\/9781315139470"},{"key":"e_1_3_2_1_3_1","volume-title":"Proceedings of OOPSLA'98 Workshop on Reflective Programming in C++ and Java","volume":"174","author":"Chiba Shigeru","year":"1998"},{"key":"e_1_3_2_1_4_1","first-page":"395","article-title":"Measuring empirical computational complexity","author":"Goldsmith Simon F","year":"2007","journal-title":"FSE. ACM"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"J. Hartmanis and R. E. Stearns. 1965. On the Computational Complexity of Algorithms. Trans. Amer. Math. Soc. 117 ( 1965 ) 285-306. http:\/\/www.jstor.org\/ stable\/1994208  J. Hartmanis and R. E. Stearns. 1965. On the Computational Complexity of Algorithms. Trans. Amer. Math. Soc. 117 ( 1965 ) 285-306. http:\/\/www.jstor.org\/ stable\/1994208","DOI":"10.1090\/S0002-9947-1965-0170805-7"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Julien Jacques and Cristian Preda. 2014. Functional data clustering: a survey. Advances in Data Analysis and Classification 8 3 ( 2014 ) 231-255.  Julien Jacques and Cristian Preda. 2014. Functional data clustering: a survey. Advances in Data Analysis and Classification 8 3 ( 2014 ) 231-255.","DOI":"10.1007\/s11634-013-0158-y"},{"key":"e_1_3_2_1_7_1","first-page":"2511","article-title":"POSTER: AFL-based Fuzzing for Java with Kelinci","author":"Kersten Rody","year":"2017","journal-title":"CCS. ACM"},{"key":"e_1_3_2_1_8_1","first-page":"254","article-title":"Perfuzz: Automatically generating pathological inputs","author":"Lemieux Caroline","year":"2018","journal-title":"ISSTA. ACM"},{"key":"e_1_3_2_1_9_1","unstructured":"libFuzzer. 2016. A library for coverage-guided fuzz testing (part of LLVM 3.9). http:\/\/llvm.org\/docs\/LibFuzzer.html. Online.  libFuzzer. 2016. A library for coverage-guided fuzz testing (part of LLVM 3.9). http:\/\/llvm.org\/docs\/LibFuzzer.html. Online."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.1982.1056489"},{"key":"e_1_3_2_1_11_1","volume-title":"Pasareanu","author":"Nilizadeh Shirin","year":"2019"},{"key":"e_1_3_2_1_12_1","unstructured":"Travis Oliphant. 2006-. NumPy: A guide to NumPy. http:\/\/www.numpy.org\/.  Travis Oliphant. 2006-. NumPy: A guide to NumPy. http:\/\/www.numpy.org\/."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2737924.2737966"},{"key":"e_1_3_2_1_14_1","unstructured":"Zooko O'Whielacronx. 2018. A program\/module to trace Python program or function execution. https:\/\/github.com\/python\/cpython\/blob\/2.7\/Lib\/trace.py. Online.  Zooko O'Whielacronx. 2018. A program\/module to trace Python program or function execution. https:\/\/github.com\/python\/cpython\/blob\/2.7\/Lib\/trace.py. Online."},{"key":"e_1_3_2_1_15_1","volume-title":"Scikit-learn: Machine Learning in Python. Journal of Machine Learning Research 12 ( 2011 ), 2825-2830.","author":"Pedregosa F.","year":"2011"},{"key":"e_1_3_2_1_16_1","first-page":"2155","article-title":"Slowfuzz: Automated domain-independent detection of algorithmic complexity vulnerabilities","author":"Petsios Theofilos","year":"2017","journal-title":"CCS. ACM"},{"key":"e_1_3_2_1_17_1","volume-title":"Functional data analysis","author":"Ramsay James O"},{"key":"e_1_3_2_1_18_1","unstructured":"Scikit-learn. 2017. Trees with MAE criterion are slow to train. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/9626. Online.  Scikit-learn. 2017. Trees with MAE criterion are slow to train. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/9626. Online."},{"key":"e_1_3_2_1_19_1","unstructured":"Scikit-learn. 2018. Sqeuclidean metric is much slower than euclidean. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/12600. Online.  Scikit-learn. 2018. Sqeuclidean metric is much slower than euclidean. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/12600. Online."},{"key":"e_1_3_2_1_20_1","unstructured":"Scikit-learn. 2019. Decision Tree Classifier. https:\/\/scikitlearn.org\/stable\/modules\/tree.html. Online.  Scikit-learn. 2019. Decision Tree Classifier. https:\/\/scikitlearn.org\/stable\/modules\/tree.html. Online."},{"key":"e_1_3_2_1_21_1","unstructured":"Scikit-learn. 2019. Gaussian Process Classifier in scikit-learn: description. https:\/\/scikit-learn.org\/stable\/modules\/gaussian_process. html#gaussianprocess-classification-gpc. Online.  Scikit-learn. 2019. Gaussian Process Classifier in scikit-learn: description. https:\/\/scikit-learn.org\/stable\/modules\/gaussian_process. html#gaussianprocess-classification-gpc. Online."},{"key":"e_1_3_2_1_22_1","unstructured":"Scikit-learn. 2019. Gaussian Process Classiifer in scikit-learn: implementations. https:\/\/scikitlearn.org\/stable\/modules\/generated\/sklearn.gaussian_process. GaussianProcess Classifier.html. Online.  Scikit-learn. 2019. Gaussian Process Classiifer in scikit-learn: implementations. https:\/\/scikitlearn.org\/stable\/modules\/generated\/sklearn.gaussian_process. GaussianProcess Classifier.html. Online."},{"key":"e_1_3_2_1_23_1","unstructured":"Scikit-learn. 2019. Linear and quadratic discriminant analysis. https:\/\/scikitlearn.org\/stable\/modules\/lda_qda.html. Online.  Scikit-learn. 2019. Linear and quadratic discriminant analysis. https:\/\/scikitlearn.org\/stable\/modules\/lda_qda.html. Online."},{"key":"e_1_3_2_1_24_1","unstructured":"Scikit-learn. 2019. Logistic Regression in scikit-learn. https:\/\/scikitlearn.org\/stable\/modules\/linear_model.html#logistic-regression. Online.  Scikit-learn. 2019. Logistic Regression in scikit-learn. https:\/\/scikitlearn.org\/stable\/modules\/linear_model.html#logistic-regression. Online."},{"key":"e_1_3_2_1_25_1","unstructured":"Scikit-learn. 2019. Make Multilabel Classification. sklearn.datasets.make_multilabel_classification.  Scikit-learn. 2019. Make Multilabel Classification. sklearn.datasets.make_multilabel_classification."},{"key":"e_1_3_2_1_26_1","unstructured":"Scikit-learn. 2019. Mini-batch KMeans. sklearn.cluster.MiniBatchKMeans.  Scikit-learn. 2019. Mini-batch KMeans. sklearn.cluster.MiniBatchKMeans."},{"key":"e_1_3_2_1_27_1","unstructured":"Scikit-learn. 2019. Performance of Logistic Regression with saga. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/13316. Online.  Scikit-learn. 2019. Performance of Logistic Regression with saga. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/13316. Online."},{"key":"e_1_3_2_1_28_1","unstructured":"Scikit-learn. 2019. Random Forest Regressor. sklearn.ensemble.RandomForestRegressor.  Scikit-learn. 2019. Random Forest Regressor. sklearn.ensemble.RandomForestRegressor."},{"key":"e_1_3_2_1_29_1","unstructured":"Scikit-learn. 2020. Performance bug in logistic regression with newton-cg. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/16186. Online.  Scikit-learn. 2020. Performance bug in logistic regression with newton-cg. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/16186. Online."},{"key":"e_1_3_2_1_30_1","unstructured":"Scikit-learn. 2020. Performance bug in Make Classification Data Set Util. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/16001. Online.  Scikit-learn. 2020. Performance bug in Make Classification Data Set Util. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/16001. Online."},{"key":"e_1_3_2_1_31_1","unstructured":"Scikit-learn. 2020. Performance bug in Make Classification Data Set Util fixed. https:\/\/github.com\/scikit-learn\/scikit-learn\/pull\/16006\/files. Online.  Scikit-learn. 2020. Performance bug in Make Classification Data Set Util fixed. https:\/\/github.com\/scikit-learn\/scikit-learn\/pull\/16006\/files. Online."},{"key":"e_1_3_2_1_32_1","unstructured":"Scikit-learn. 2020. Performance bug in regression with newton-cg fixed. https:\/\/github.com\/scikit-learn\/scikit-learn\/pull\/16266\/files. Online.  Scikit-learn. 2020. Performance bug in regression with newton-cg fixed. https:\/\/github.com\/scikit-learn\/scikit-learn\/pull\/16266\/files. Online."},{"key":"e_1_3_2_1_33_1","unstructured":"Scikit-learn. 2020. Performance bug in Util Batch Generator module. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/16158. Online.  Scikit-learn. 2020. Performance bug in Util Batch Generator module. https:\/\/github.com\/scikit-learn\/scikit-learn\/issues\/16158. Online."},{"key":"e_1_3_2_1_34_1","unstructured":"Scikit-learn. 2020. Performance bug in Util Batch Generator module fixed. https:\/\/github.com\/scikit-learn\/scikit-learn\/pull\/16181\/files. Online.  Scikit-learn. 2020. Performance bug in Util Batch Generator module fixed. https:\/\/github.com\/scikit-learn\/scikit-learn\/pull\/16181\/files. Online."},{"key":"e_1_3_2_1_35_1","volume-title":"Algorithms","author":"Sedgewick Robert","edition":"4"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Linhai Song and Shan Lu. 2014. Statistical debugging for real-world performance problems. OOPSLA 49 10 ( 2014 ) 561-578.  Linhai Song and Shan Lu. 2014. Statistical debugging for real-world performance problems. OOPSLA 49 10 ( 2014 ) 561-578.","DOI":"10.1145\/2714064.2660234"},{"key":"e_1_3_2_1_37_1","unstructured":"Tensorflow. 2019. Transpose can be very slow on CPU. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/27383. Online.  Tensorflow. 2019. Transpose can be very slow on CPU. https:\/\/github.com\/tensorflow\/tensorflow\/issues\/27383. Online."},{"key":"e_1_3_2_1_38_1","first-page":"2468","article-title":"Diferential Performance Debugging with Discriminant Regression Trees","author":"Tizpaz-Niari Saeid","year":"2018","journal-title":"AAAI."},{"key":"e_1_3_2_1_39_1","volume-title":"Sriram Sankaranarayanan, and Ashutosh Trivedi.","author":"Tizpaz-Niari Saeid","year":"2017"},{"key":"e_1_3_2_1_40_1","volume-title":"Data-Driven Debugging for Functional Side Channels. https:\/\/arxiv.org\/abs\/","author":"Tizpaz-Niari Saeid","year":"1808"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2018.00038"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2521368"},{"key":"e_1_3_2_1_43_1","volume-title":"The Fuzzing Book","author":"Zeller Andreas","year":"2019"}],"event":{"name":"ISSTA '20: 29th ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Virtual Event USA","acronym":"ISSTA '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 29th ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3395363.3404540","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3395363.3404540","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:38:45Z","timestamp":1750199925000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3395363.3404540"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,7,18]]},"references-count":43,"alternative-id":["10.1145\/3395363.3404540","10.1145\/3395363"],"URL":"https:\/\/doi.org\/10.1145\/3395363.3404540","relation":{},"subject":[],"published":{"date-parts":[[2020,7,18]]},"assertion":[{"value":"2020-07-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}