{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T16:38:16Z","timestamp":1775579896915,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":108,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,21]],"date-time":"2022-05-21T00:00:00Z","timestamp":1653091200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"ARO","award":["W911NF1920321"],"award-info":[{"award-number":["W911NF1920321"]}]},{"name":"DOE Early Career Award","award":["DESC0014195 0003"],"award-info":[{"award-number":["DESC0014195 0003"]}]},{"name":"NSF","award":["CNS1764039, CNS1956180, CCF1837120, CCF2119184, CNS1952050, CCF1823032"],"award-info":[{"award-number":["CNS1764039, CNS1956180, CCF1837120, CCF2119184, CNS1952050, CCF1823032"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,21]]},"DOI":"10.1145\/3510003.3510068","type":"proceedings-article","created":{"date-parts":[[2022,7,5]],"date-time":"2022-07-05T22:42:59Z","timestamp":1657060979000},"page":"212-224","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["Automated testing of software that uses machine learning APIs"],"prefix":"10.1145","author":[{"given":"Chengcheng","family":"Wan","sequence":"first","affiliation":[{"name":"University of Chicago"}]},{"given":"Shicheng","family":"Liu","sequence":"additional","affiliation":[{"name":"University of Chicago"}]},{"given":"Sophie","family":"Xie","sequence":"additional","affiliation":[{"name":"Whitney Young High School"}]},{"given":"Yifan","family":"Liu","sequence":"additional","affiliation":[{"name":"University of Chicago"}]},{"given":"Henry","family":"Hoffmann","sequence":"additional","affiliation":[{"name":"University of Chicago"}]},{"given":"Michael","family":"Maire","sequence":"additional","affiliation":[{"name":"University of Chicago"}]},{"given":"Shan","family":"Lu","sequence":"additional","affiliation":[{"name":"University of Chicago"}]}],"member":"320","published-online":{"date-parts":[[2022,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Google \"Google cloud ai.\" Online document https:\/\/cloud.google.com\/products\/ai 2020."},{"key":"e_1_3_2_1_2_1","unstructured":"Amazon \"Amazon artificial intelligence service.\" Online document https:\/\/aws.amazon.com\/machine-learning\/ai-services 2020."},{"key":"e_1_3_2_1_3_1","unstructured":"Microsoft \"Microsoft azure cognitive services.\" Online document https:\/\/azure.microsoft.com\/en-us\/services\/cognitive-services 2020."},{"key":"e_1_3_2_1_4_1","unstructured":"IBM \"Ibm watson.\" Online document https:\/\/www.ibm.com\/watson 2020."},{"key":"e_1_3_2_1_5_1","first-page":"291","volume-title":"Software engineering for machine learning: A case study,\" in ICSE-SEIP","author":"Amershi S.","year":"2019","unstructured":"S. Amershi, A. Begel, C. Bird, R. DeLine, H. Gall, E. Kamar, N. Nagappan, B. Nushi, and T. Zimmermann, \"Software engineering for machine learning: A case study,\" in ICSE-SEIP, pp. 291--300, IEEE, 2019."},{"issue":"2","key":"e_1_3_2_1_6_1","first-page":"1301","article-title":"A survey on machine learning: concept, algorithms and applications","volume":"5","author":"Das K.","year":"2017","unstructured":"K. Das and R. N. Behera, \"A survey on machine learning: concept, algorithms and applications,\" International Journal of Innovative Research in Computer and Communication Engineering, vol. 5, no. 2, pp. 1301--1309, 2017.","journal-title":"International Journal of Innovative Research in Computer and Communication Engineering"},{"key":"e_1_3_2_1_7_1","volume-title":"Are machine learning cloud apis used correctly?,\" in 43th International Conference on Software Engineering (ICSE'21)","author":"Wan C.","year":"2021","unstructured":"C. Wan, S. Liu, H. Hoffmann, M. Maire, and S. Lu, \"Are machine learning cloud apis used correctly?,\" in 43th International Conference on Software Engineering (ICSE'21), 2021."},{"key":"e_1_3_2_1_8_1","unstructured":"Phoenix \"A fire-detection application.\" https:\/\/github.com\/yunusemreemik\/Phoenix."},{"key":"e_1_3_2_1_9_1","volume-title":"Tensorfuzz: Debugging neural networks with coverage-guided fuzzing,\" in ICML","author":"Odena A.","year":"2019","unstructured":"A. Odena, C. Olsson, D. Andersen, and I. Goodfellow, \"Tensorfuzz: Debugging neural networks with coverage-guided fuzzing,\" in ICML, 2019."},{"key":"e_1_3_2_1_10_1","volume-title":"Deephunter: Hunting deep neural network defects via coverage-guided fuzzing,\" arXiv preprint arXiv:1809.01266","author":"Xie X.","year":"2018","unstructured":"X. Xie, L. Ma, F. Juefei-Xu, H. Chen, M. Xue, B. Li, Y. Liu, J. Zhao, J. Yin, and S. See, \"Deephunter: Hunting deep neural network defects via coverage-guided fuzzing,\" arXiv preprint arXiv:1809.01266, 2018."},{"key":"e_1_3_2_1_11_1","first-page":"458","article-title":"Metamorphic testing and certified mitigation of fairness violations in nlp models","author":"Ma P.","year":"2020","unstructured":"P. Ma, S. Wang, and J. Liu, \"Metamorphic testing and certified mitigation of fairness violations in nlp models.\" in IJCAI, pp. 458--465, 2020.","journal-title":"IJCAI"},{"key":"e_1_3_2_1_12_1","first-page":"1110","volume-title":"Automatic test oracle for image processing applications using support vector machines,\" in 2015 6th IEEE International Conference on Software Engineering and Service Science (ICSESS)","author":"Jameel T.","year":"2015","unstructured":"T. Jameel, L. Mengxiang, and L. Chao, \"Automatic test oracle for image processing applications using support vector machines,\" in 2015 6th IEEE International Conference on Software Engineering and Service Science (ICSESS), pp. 1110--1113, IEEE, 2015."},{"key":"e_1_3_2_1_13_1","first-page":"1","volume-title":"Feature-based test oracles to categorize synthetic 3d and 2d images of blood vessels,\" in Proceedings of the 2nd Brazilian Symposium on Systematic and Automated Software Testing","author":"J\u00fanior M. C.","year":"2017","unstructured":"M. C. J\u00fanior, R. A. Oliveira, M. A. Valverde, M. P. Jackowski, F. L. Nunes, and M. E. Delamaro, \"Feature-based test oracles to categorize synthetic 3d and 2d images of blood vessels,\" in Proceedings of the 2nd Brazilian Symposium on Systematic and Automated Software Testing, pp. 1--6, 2017."},{"key":"e_1_3_2_1_14_1","volume-title":"Reliability and Security Companion (QRS-C)","author":"Jiang C.","year":"2018","unstructured":"C. Jiang, S. Huang, and Z. Hui, \"Metamorphic testing of image region growth programs in image processing applications,\" in 2018 IEEE International Conference on Software Quality, Reliability and Security Companion (QRS-C), 2018."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"M. Srinivasan M. P. Shahri I. Kahanda and U. Kanewala \"Quality assurance of bioinformatics software: a case study of testing a biomedical text processing tool using metamorphic testing \" in Proceedings of the 3rd International Workshop on Metamorphic Testing pp. 26--33 2018.","DOI":"10.1145\/3193977.3193981"},{"key":"e_1_3_2_1_16_1","volume-title":"Adversarial sample detection for deep neural network through model mutation testing,\" in ICSE","author":"Wang J.","year":"2019","unstructured":"J. Wang, G. Dong, J. Sun, X. Wang, and P. Zhang, \"Adversarial sample detection for deep neural network through model mutation testing,\" in ICSE, 2019."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132747.3132785"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_19_1","first-page":"146","volume-title":"Deephunter: A coverage-guided fuzz testing framework for deep neural networks,\" in ISSTA","author":"Xie X.","year":"2019","unstructured":"X. Xie, L. Ma, F. Juefei-Xu, M. Xue, H. Chen, Y. Liu, J. Zhao, B. Li, J. Yin, and S. See, \"Deephunter: A coverage-guided fuzz testing framework for deep neural networks,\" in ISSTA, pp. 146--157, 2019."},{"key":"e_1_3_2_1_20_1","volume-title":"Mode: automated neural network model debugging via state differential analysis and input selection,\" in ESEC\/FSE","author":"Ma S.","year":"2018","unstructured":"S. Ma, Y. Liu, W.-C. Lee, X. Zhang, and A. Grama, \"Mode: automated neural network model debugging via state differential analysis and input selection,\" in ESEC\/FSE, 2018."},{"key":"e_1_3_2_1_21_1","volume-title":"Lamp: data provenance for graph based machine learning algorithms through derivative computation,\" in FSE","author":"Ma S.","year":"2017","unstructured":"S. Ma, Y. Aafer, Z. Xu, W.-C. Lee, J. Zhai, Y. Liu, and X. Zhang, \"Lamp: data provenance for graph based machine learning algorithms through derivative computation,\" in FSE, 2017."},{"key":"e_1_3_2_1_22_1","volume-title":"Autofocus: interpreting attention-based neural networks by code perturbation,\" in ASE","author":"Bui N. D.","year":"2019","unstructured":"N. D. Bui, Y. Yu, and L. Jiang, \"Autofocus: interpreting attention-based neural networks by code perturbation,\" in ASE, 2019."},{"key":"e_1_3_2_1_23_1","volume-title":"Testing vision-based control systems using learnable evolutionary algorithms,\" in ICSE","author":"Abdessalem R. B.","year":"2018","unstructured":"R. B. Abdessalem, S. Nejati, L. C. Briand, and T. Stifter, \"Testing vision-based control systems using learnable evolutionary algorithms,\" in ICSE, 2018."},{"key":"e_1_3_2_1_24_1","volume-title":"Deepmutation: Mutation testing of deep learning systems,\" in ISSRE","author":"Ma L.","year":"2018","unstructured":"L. Ma, F. Zhang, J. Sun, M. Xue, B. Li, F. Juefei-Xu, C. Xie, L. Li, Y. Liu, J. Zhao, et al., \"Deepmutation: Mutation testing of deep learning systems,\" in ISSRE, 2018."},{"key":"e_1_3_2_1_25_1","volume-title":"Deeproad: Gan-based metamorphic testing and input validation framework for autonomous driving systems,\" in ASE","author":"Zhang M.","year":"2018","unstructured":"M. Zhang, Y. Zhang, L. Zhang, C. Liu, and S. Khurshid, \"Deeproad: Gan-based metamorphic testing and input validation framework for autonomous driving systems,\" in ASE, 2018."},{"key":"e_1_3_2_1_26_1","volume-title":"Identifying implementation bugs in machine learning based image classifiers using metamorphic testing,\" in ISSTA","author":"Dwarakanath A.","year":"2018","unstructured":"A. Dwarakanath, M. Ahuja, S. Sikand, R. M. Rao, R. J. C. Bose, N. Dubash, and S. Podder, \"Identifying implementation bugs in machine learning based image classifiers using metamorphic testing,\" in ISSTA, 2018."},{"key":"e_1_3_2_1_27_1","volume-title":"Fairness testing: testing software for discrimination,\" in FSE","author":"Galhotra S.","year":"2017","unstructured":"S. Galhotra, Y. Brun, and A. Meliou, \"Fairness testing: testing software for discrimination,\" in FSE, 2017."},{"key":"e_1_3_2_1_28_1","volume-title":"Themis: Automatically testing software for discrimination,\" in ESEC\/FSE","author":"Angell R.","year":"2018","unstructured":"R. Angell, B. Johnson, Y. Brun, and A. Meliou, \"Themis: Automatically testing software for discrimination,\" in ESEC\/FSE, 2018."},{"key":"e_1_3_2_1_29_1","volume-title":"Model-tracker: Redesigning performance analysis tools for machine learning,\" in CHI","author":"Amershi S.","year":"2015","unstructured":"S. Amershi, M. Chickering, S. M. Drucker, B. Lee, P. Simard, and J. Suh, \"Model-tracker: Redesigning performance analysis tools for machine learning,\" in CHI, 2015."},{"key":"e_1_3_2_1_30_1","volume-title":"Correlations between deep neural network model coverage criteria and model quality,\" in ESEC\/FSE","author":"Yan S.","year":"2020","unstructured":"S. Yan, G. Tao, X. Liu, J. Zhai, S. Ma, L. Xu, and X. Zhang, \"Correlations between deep neural network model coverage criteria and model quality,\" in ESEC\/FSE, 2020."},{"key":"e_1_3_2_1_31_1","volume-title":"Deepsearch: A simple and effective blackbox attack for deep neural networks,\" in ESEC\/FSE","author":"Zhang F.","year":"2020","unstructured":"F. Zhang, S. P. Chowdhury, and M. Christakis, \"Deepsearch: A simple and effective blackbox attack for deep neural networks,\" in ESEC\/FSE, 2020."},{"key":"e_1_3_2_1_32_1","volume-title":"Is neuron coverage a meaningful measure for testing deep neural networks?,\" in ESEC\/FSE","author":"Harel-Canada F.","year":"2020","unstructured":"F. Harel-Canada, L. Wang, M. A. Gulzar, Q. Gu, and M. Kim, \"Is neuron coverage a meaningful measure for testing deep neural networks?,\" in ESEC\/FSE, 2020."},{"key":"e_1_3_2_1_33_1","volume-title":"Model-based exploration of the frontier of behaviours for deep learning system testing,\" in ESEC\/FSE","author":"Riccio V.","year":"2020","unstructured":"V. Riccio and P. Tonella, \"Model-based exploration of the frontier of behaviours for deep learning system testing,\" in ESEC\/FSE, 2020."},{"key":"e_1_3_2_1_34_1","volume-title":"Importance-driven deep learning system testing,\" in ICSE","author":"Gerasimou S.","year":"2020","unstructured":"S. Gerasimou, H. F. Eniser, A. Sen, and A. Cakan, \"Importance-driven deep learning system testing,\" in ICSE, 2020."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380337"},{"key":"e_1_3_2_1_36_1","volume-title":"Towards characterizing adversarial defects of deep learning software from the lens of uncertainty,\" in ICSE","author":"Zhang X.","year":"2020","unstructured":"X. Zhang, X. Xie, L. Ma, X. Du, Q. Hu, Y. Liu, J. Zhao, and M. Sun, \"Towards characterizing adversarial defects of deep learning software from the lens of uncertainty,\" in ICSE, 2020."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416609"},{"key":"e_1_3_2_1_38_1","volume-title":"Deepgini: prioritizing massive tests to enhance the robustness of deep neural networks,\" in ISSTA","author":"Feng Y.","year":"2020","unstructured":"Y. Feng, Q. Shi, X. Gao, J. Wan, C. Fang, and Z. Chen, \"Deepgini: prioritizing massive tests to enhance the robustness of deep neural networks,\" in ISSTA, 2020."},{"key":"e_1_3_2_1_39_1","volume-title":"Detecting flaky tests in probabilistic and machine learning applications,\" in ISSTA","author":"Dutta S.","year":"2020","unstructured":"S. Dutta, A. Shi, R. Choudhary, Z. Zhang, A. Jain, and S. Misailovic, \"Detecting flaky tests in probabilistic and machine learning applications,\" in ISSTA, 2020."},{"key":"e_1_3_2_1_40_1","volume-title":"Effective white-box testing of deep neural networks with adaptive neuron-selection strategy,\" in ISSTA","author":"Lee S.","year":"2020","unstructured":"S. Lee, S. Cha, D. Lee, and H. Oh, \"Effective white-box testing of deep neural networks with adaptive neuron-selection strategy,\" in ISSTA, 2020."},{"key":"e_1_3_2_1_41_1","volume-title":"Higher income, larger loan? monotonicity testing of machine learning models,\" in ISSTA","author":"Sharma A.","year":"2020","unstructured":"A. Sharma and H. Wehrheim, \"Higher income, larger loan? monotonicity testing of machine learning models,\" in ISSTA, 2020."},{"key":"e_1_3_2_1_42_1","volume-title":"Apricot: a weight-adaptation approach to fixing deep learning models,\" in ASE","author":"Zhang H.","year":"2019","unstructured":"H. Zhang and W. Chan, \"Apricot: a weight-adaptation approach to fixing deep learning models,\" in ASE, 2019."},{"key":"e_1_3_2_1_43_1","volume-title":"Operational calibration: Debugging confidence errors for dnns in the field,\" in ESEC\/FSE","author":"Li Z.","year":"2020","unstructured":"Z. Li, X. Ma, C. Xu, J. Xu, C. Cao, and J. L\u00fc, \"Operational calibration: Debugging confidence errors for dnns in the field,\" in ESEC\/FSE, 2020."},{"key":"e_1_3_2_1_44_1","volume-title":"Automatic testing and improvement of machine translation,\" in ICSE","author":"Sun Z.","year":"2020","unstructured":"Z. Sun, J. M. Zhang, M. Harman, M. Papadakis, and L. Zhang, \"Automatic testing and improvement of machine translation,\" in ICSE, 2020."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380378"},{"key":"e_1_3_2_1_46_1","unstructured":"Microsoft \"Visual studio code.\" Online document https:\/\/code.visualstudio.com\/ 2021."},{"key":"e_1_3_2_1_47_1","first-page":"26","article-title":"Deconstructing dynamic symbolic execution","volume":"40","author":"Irlbeck M.","year":"2015","unstructured":"M. Irlbeck et al., \"Deconstructing dynamic symbolic execution,\" Dependable Software Systems Engineering, vol. 40, p. 26, 2015.","journal-title":"Dependable Software Systems Engineering"},{"key":"e_1_3_2_1_48_1","volume-title":"Meta pseudo labels,\" in CVPR","author":"Pham H.","year":"2021","unstructured":"H. Pham, Z. Dai, Q. Xie, and Q. V. Le, \"Meta pseudo labels,\" in CVPR, 2021."},{"key":"e_1_3_2_1_49_1","unstructured":"recipeGo \"A recipe recommendation application.\" https:\/\/github.com\/Reckonzz\/recipeGO."},{"key":"e_1_3_2_1_50_1","unstructured":"emotion2music \"A smart music player application \" https:\/\/github.com\/varnachandar\/emotion2music."},{"key":"e_1_3_2_1_51_1","unstructured":"NsTool \"A monitor application.\" https:\/\/github.com\/clarkwkw\/ns_online_toolkit."},{"key":"e_1_3_2_1_52_1","unstructured":"noteScript \"A lecture note application.\" https:\/\/github.com\/GalenWong\/noteScript."},{"key":"e_1_3_2_1_53_1","unstructured":"stockmine \"A stock prediction application.\" https:\/\/github.com\/nicholasadamou\/stockmine."},{"issue":"12","key":"e_1_3_2_1_54_1","first-page":"2009","article-title":"Twitter sentiment classification using distant supervision","volume":"1","author":"Go A.","year":"2009","unstructured":"A. Go, R. Bhayani, and L. Huang, \"Twitter sentiment classification using distant supervision,\" CS224N project report, Stanford, vol. 1, no. 12, p. 2009, 2009.","journal-title":"CS224N project report, Stanford"},{"key":"e_1_3_2_1_55_1","unstructured":"Klassroom \"A lecture note application.\" https:\/\/github.com\/dev5151\/Klassroom."},{"key":"e_1_3_2_1_56_1","unstructured":"TRANSLATOR \"A smart light application.\" https:\/\/github.com\/mubeenafatima\/TRANSLATOR."},{"key":"e_1_3_2_1_57_1","unstructured":"HeapSortCypher \"A garbage classification application.\" https:\/\/github.com\/matthew-chu\/heapsortcypher."},{"key":"e_1_3_2_1_58_1","unstructured":"D. Chaffey \"Search engine marketing statistics 2020.\" https:\/\/www.smartinsights.com\/search-engine-marketing\/search-engine-statistics\/."},{"key":"e_1_3_2_1_59_1","unstructured":"M. Bing \"Bing image search.\" https:\/\/www.bing.com\/images\/trending?FORM=ILPTRD."},{"key":"e_1_3_2_1_60_1","unstructured":"\"Lorem picsum.\" https:\/\/picsum.photos."},{"key":"e_1_3_2_1_61_1","unstructured":"\"Wikipedia.\" https:\/\/en.m.wikipedia.org\/."},{"key":"e_1_3_2_1_62_1","unstructured":"\"Encyclopedia britannica.\" https:\/\/www.britannica.com\/."},{"key":"e_1_3_2_1_63_1","unstructured":"\"Pillow: Python imaging library.\" https:\/\/pypi.org\/project\/Pillow\/."},{"key":"e_1_3_2_1_64_1","unstructured":"WanderStub \"An exchange conversion application.\" https:\/\/github.com\/richardjpark26\/WanderStub."},{"issue":"8","key":"e_1_3_2_1_65_1","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford A.","year":"2019","unstructured":"A. Radford, J. Wu, R. Child, D. Luan, D. Amodei, and I. Sutskever, \"Language models are unsupervised multitask learners,\" OpenAI blog, vol. 1, no. 8, p. 9, 2019.","journal-title":"OpenAI blog"},{"key":"e_1_3_2_1_66_1","unstructured":"\"pyttsx3: Text-to-speech library for python.\" https:\/\/pypi.org\/project\/pyttsx3\/."},{"key":"e_1_3_2_1_67_1","volume-title":"Imagenet: A large-scale hierarchical image database,\" in CVPR","author":"Deng J.","year":"2009","unstructured":"J. Deng, W. Dong, R. Socher, L. Li, K. Li, and L. Fei-Fei, \"Imagenet: A large-scale hierarchical image database,\" in CVPR, 2009."},{"key":"e_1_3_2_1_68_1","first-page":"1","article-title":"The open images dataset v4","author":"Kuznetsova A.","year":"2020","unstructured":"A. Kuznetsova, H. Rom, N. Alldrin, J. Uijlings, I. Krasin, J. Pont-Tuset, S. Kamali, S. Popov, M. Malloci, A. Kolesnikov, et al., \"The open images dataset v4,\" International Journal of Computer Vision, pp. 1--26, 2020.","journal-title":"International Journal of Computer Vision"},{"key":"e_1_3_2_1_69_1","volume-title":"A review on generative adversarial networks: Algorithms, theory, and applications,\" arXiv preprint arXiv:2001.06937","author":"Gui J.","year":"2020","unstructured":"J. Gui, Z. Sun, Y. Wen, D. Tao, and J. Ye, \"A review on generative adversarial networks: Algorithms, theory, and applications,\" arXiv preprint arXiv:2001.06937, 2020."},{"key":"e_1_3_2_1_70_1","volume-title":"Zero-shot text-to-image generation,\" arXiv preprint arXiv:2102.12092","author":"Ramesh A.","year":"2021","unstructured":"A. Ramesh, M. Pavlov, G. Goh, S. Gray, C. Voss, A. Radford, M. Chen, and I. Sutskever, \"Zero-shot text-to-image generation,\" arXiv preprint arXiv:2102.12092, 2021."},{"key":"e_1_3_2_1_71_1","unstructured":"FESMKMITL \"A smart camera application.\" https:\/\/github.com\/matthewjmc\/FESMKMITL."},{"key":"e_1_3_2_1_72_1","unstructured":"Verlan \"A pet application.\" https:\/\/github.com\/sarvesh-tech\/Verlan."},{"key":"e_1_3_2_1_73_1","unstructured":"FortniteKillfeed \"A real time tracker application.\" https:\/\/github.com\/Godsinred\/FortniteKillfeed."},{"key":"e_1_3_2_1_74_1","unstructured":"\"Wikipedia.\" https:\/\/www.wikidata.org\/."},{"key":"e_1_3_2_1_75_1","first-page":"171","volume-title":"CAV 2011, Snowbird, UT, USA, July 14--20, 2011. Proceedings (G. Gopalakrishnan and S. Qadeer, eds.)","volume":"6806","author":"Barrett C. W.","year":"2011","unstructured":"C. W. Barrett, C. L. Conway, M. Deters, L. Hadarean, D. Jovanovic, T. King, A. Reynolds, and C. Tinelli, \"CVC4,\" in Computer Aided Verification - 23rd International Conference, CAV 2011, Snowbird, UT, USA, July 14--20, 2011. Proceedings (G. Gopalakrishnan and S. Qadeer, eds.), vol. 6806 of Lecture Notes in Computer Science, pp. 171--177, Springer, 2011."},{"key":"e_1_3_2_1_76_1","unstructured":"\"Python system-specific parameters and functions.\" https:\/\/docs.python.org\/3\/library\/sys.html#sys.settrace."},{"key":"e_1_3_2_1_77_1","unstructured":"D. Marby and N. Yonskai \"Pyan3: Offline call graph generator for python 3.\" https:\/\/github.com\/davidfraser\/pyan."},{"key":"e_1_3_2_1_78_1","unstructured":"D. Halter \"Jedi: an awesome auto-completion static analysis and refactoring library for python.\" Online document https:\/\/jedi.readthedocs.io."},{"key":"e_1_3_2_1_79_1","unstructured":"\"scikit-learn: Machine learning in python.\" https:\/\/scikit-learn.org\/stable\/."},{"key":"e_1_3_2_1_80_1","unstructured":"C. Wan S. Liu S. Xie Y. Liu H. Hoffmann M. Maire and S. Lu \"Project Webpage: Accurate Learning for EneRgy and Timeliness in Software System.\" https:\/\/alert.cs.uchicago.edu\/#release."},{"key":"e_1_3_2_1_81_1","unstructured":"Kaggle \"Twitter us airline sentiment.\" https:\/\/www.kaggle.com\/crowdflower\/twitter-airline-sentiment."},{"key":"e_1_3_2_1_82_1","unstructured":"\"100 english daily sentences for daily use.\" https:\/\/englishspeakingcourse.net\/100-english-sentences-for-daily-use\/."},{"key":"e_1_3_2_1_83_1","unstructured":"Fuzzit.dev \"Pythonfuzz: coverage-guided fuzz testing for python.\" https:\/\/gitlab.com\/gitlab-org\/security-products\/analyzers\/fuzzers\/pythonfuzz."},{"key":"e_1_3_2_1_84_1","volume-title":"Trials and tribulations of developers of intelligent systems: A field study,\" in VL\/HCC","author":"Hill C.","year":"2016","unstructured":"C. Hill, R. Bellamy, T. Erickson, and M. Burnett, \"Trials and tribulations of developers of intelligent systems: A field study,\" in VL\/HCC, 2016."},{"key":"e_1_3_2_1_85_1","volume-title":"The emerging role of data scientists on software development teams,\" in ICSE","author":"Kim M.","year":"2016","unstructured":"M. Kim, T. Zimmermann, R. DeLine, and A. Begel, \"The emerging role of data scientists on software development teams,\" in ICSE, 2016."},{"key":"e_1_3_2_1_86_1","volume-title":"Data scientists in software teams: State of the art and challenges,\" TSE","author":"Kim M.","year":"2017","unstructured":"M. Kim, T. Zimmermann, R. DeLine, and A. Begel, \"Data scientists in software teams: State of the art and challenges,\" TSE, 2017."},{"key":"e_1_3_2_1_87_1","first-page":"14","volume-title":"Reliability and Security Companion (QRS-C)","author":"Zhao X.","year":"2018","unstructured":"X. Zhao and X. Gao, \"An ai software test method based on scene deductive approach,\" in 2018 IEEE International Conference on Software Quality, Reliability and Security Companion (QRS-C), pp. 14--20, IEEE, 2018."},{"key":"e_1_3_2_1_88_1","volume-title":"Testing of autonomous systems-challenges and current state-of-the-art,\" in INCOSE international symposium","author":"Helle P.","year":"2016","unstructured":"P. Helle, W. Schamai, and C. Strobel, \"Testing of autonomous systems-challenges and current state-of-the-art,\" in INCOSE international symposium, 2016."},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"crossref","unstructured":"T. Linz \"Testing autonomous systems \" in The Future of Software Quality Assurance pp. 61--75 Springer Cham 2020.","DOI":"10.1007\/978-3-030-29509-7_5"},{"key":"e_1_3_2_1_90_1","volume-title":"Metamorphic model-based testing of autonomous systems,\" in 2017 IEEE\/ACM 2nd International Workshop on Metamorphic Testing (MET)","author":"Lindvall M.","year":"2017","unstructured":"M. Lindvall, A. Porter, G. Magnusson, and C. Schulze, \"Metamorphic model-based testing of autonomous systems,\" in 2017 IEEE\/ACM 2nd International Workshop on Metamorphic Testing (MET), 2017."},{"key":"e_1_3_2_1_91_1","volume-title":"Learning-based testing for autonomous systems using spatial and temporal requirements,\" in Proceedings of the 1st International Workshop on Machine Learning and Software Engineering in Symbiosis","author":"Khosrowjerdi H.","year":"2018","unstructured":"H. Khosrowjerdi and K. Meinke, \"Learning-based testing for autonomous systems using spatial and temporal requirements,\" in Proceedings of the 1st International Workshop on Machine Learning and Software Engineering in Symbiosis, 2018."},{"key":"e_1_3_2_1_92_1","volume-title":"Datamorphic testing: A method for testing intelligent applications,\" in 2019 IEEE International Conference On Artificial Intelligence Testing (AITest)","author":"Zhu H.","year":"2019","unstructured":"H. Zhu, D. Liu, I. Bayley, R. Harrison, and F. Cuzzolin, \"Datamorphic testing: A method for testing intelligent applications,\" in 2019 IEEE International Conference On Artificial Intelligence Testing (AITest), 2019."},{"key":"e_1_3_2_1_93_1","volume-title":"Detecting numerical bugs in neural network architectures,\" in ESEC\/FSE","author":"Zhang Y.","year":"2020","unstructured":"Y. Zhang, L. Ren, L. Chen, Y. Xiong, S.-C. Cheung, and T. Xie, \"Detecting numerical bugs in neural network architectures,\" in ESEC\/FSE, 2020."},{"key":"e_1_3_2_1_94_1","volume-title":"Taxonomy of real faults in deep learning systems,\" in ICSE","author":"Jahangirova G.","year":"2020","unstructured":"G. Jahangirova, N. Humbatova, G. Bavota, V. Riccio, A. Stocco, and P. Tonella, \"Taxonomy of real faults in deep learning systems,\" in ICSE, 2020."},{"key":"e_1_3_2_1_95_1","volume-title":"How do api selections affect the runtime performance of data analytics tasks?,\" in ASE","author":"Tao Y.","year":"2019","unstructured":"Y. Tao, S. Tang, Y. Liu, Z. Xu, and S. Qin, \"How do api selections affect the runtime performance of data analytics tasks?,\" in ASE, 2019."},{"key":"e_1_3_2_1_96_1","first-page":"313","volume-title":"Reliability and Security (QRS)","author":"Cheng D.","year":"2018","unstructured":"D. Cheng, C. Cao, C. Xu, and X. Ma, \"Manifesting bugs in machine learning code: An explorative study with mutation testing,\" in 2018 IEEE International Conference on Software Quality, Reliability and Security (QRS), pp. 313--324, IEEE, 2018."},{"key":"e_1_3_2_1_97_1","volume-title":"Cradle: cross-backend validation to detect and localize bugs in deep learning libraries,\" in ICSE","author":"Pham H. V.","year":"2019","unstructured":"H. V. Pham, T. Lutellier, W. Qi, and L. Tan, \"Cradle: cross-backend validation to detect and localize bugs in deep learning libraries,\" in ICSE, 2019."},{"key":"e_1_3_2_1_98_1","volume-title":"Api design for machine learning software: experiences from the scikit-learn project,\" arXiv preprint arXiv:1309.0238","author":"Buitinck L.","year":"2013","unstructured":"L. Buitinck, G. Louppe, M. Blondel, F. Pedregosa, A. Mueller, O. Grisel, V. Niculae, P. Prettenhofer, A. Gramfort, J. Grobler, et al., \"Api design for machine learning software: experiences from the scikit-learn project,\" arXiv preprint arXiv:1309.0238, 2013."},{"key":"e_1_3_2_1_99_1","volume-title":"Mli: An api for distributed machine learning,\" in ICDM","author":"Sparks E. R.","year":"2013","unstructured":"E. R. Sparks, A. Talwalkar, V. Smith, J. Kottalam, X. Pan, J. Gonzalez, M. J. Franklin, M. I. Jordan, and T. Kraska, \"Mli: An api for distributed machine learning,\" in ICDM, 2013."},{"key":"e_1_3_2_1_100_1","volume-title":"Comparative study of deep learning software frameworks,\" arXiv preprint arXiv:1511.06435","author":"Bahrampour S.","year":"2015","unstructured":"S. Bahrampour, N. Ramakrishnan, L. Schott, and M. Shah, \"Comparative study of deep learning software frameworks,\" arXiv preprint arXiv:1511.06435, 2015."},{"key":"e_1_3_2_1_101_1","volume-title":"A study of oracle approximations in testing deep learning libraries,\" in ASE","author":"Nejadgholi M.","year":"2019","unstructured":"M. Nejadgholi and J. Yang, \"A study of oracle approximations in testing deep learning libraries,\" in ASE, 2019."},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416571"},{"key":"e_1_3_2_1_103_1","volume-title":"Detecting and understanding real-world differential performance bugs in machine learning libraries,\" in ISSTA","author":"Tizpaz-Niari S.","year":"2020","unstructured":"S. Tizpaz-Niari, P. Cern\u1ef3, and A. Trivedi, \"Detecting and understanding real-world differential performance bugs in machine learning libraries,\" in ISSTA, 2020."},{"key":"e_1_3_2_1_104_1","first-page":"157","article-title":"Are rest apis for cloud computing well-designed? an exploratory study","author":"Petrillo F.","year":"2016","unstructured":"F. Petrillo, P. Merle, N. Moha, and Y.-G. Gu\u00e9h\u00e9neuc, \"Are rest apis for cloud computing well-designed? an exploratory study,\" in ICSOC, pp. 157--170, Springer, 2016.","journal-title":"ICSOC"},{"key":"e_1_3_2_1_105_1","volume-title":"Aflow-ml: A restful api for machine-learning predictions of materials properties,\" Computational Materials Science","author":"Gossett E.","year":"2018","unstructured":"E. Gossett, C. Toher, C. Oses, O. Isayev, F. Legrain, F. Rose, E. Zurek, J. Carrete, N. Mingo, A. Tropsha, et al., \"Aflow-ml: A restful api for machine-learning predictions of materials properties,\" Computational Materials Science, 2018."},{"key":"e_1_3_2_1_106_1","volume-title":"Differential regression testing for rest apis,\" in ISSTA","author":"Godefroid P.","year":"2020","unstructured":"P. Godefroid, D. Lehmann, and M. Polishchuk, \"Differential regression testing for rest apis,\" in ISSTA, 2020."},{"key":"e_1_3_2_1_107_1","first-page":"141","volume-title":"Verification and Validation","author":"McMinn P.","year":"2012","unstructured":"P. McMinn, M. Shahbaz, and M. Stevenson, \"Search-based test input generation for string data types using the results of web queries,\" in 2012 IEEE Fifth International Conference on Software Testing, Verification and Validation, pp. 141--150, IEEE, 2012."},{"key":"e_1_3_2_1_108_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.scico.2014.04.008"}],"event":{"name":"ICSE '22: 44th International Conference on Software Engineering","location":"Pittsburgh Pennsylvania","acronym":"ICSE '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 44th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510068","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510068","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3510003.3510068","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:04Z","timestamp":1750191124000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3510003.3510068"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,21]]},"references-count":108,"alternative-id":["10.1145\/3510003.3510068","10.1145\/3510003"],"URL":"https:\/\/doi.org\/10.1145\/3510003.3510068","relation":{},"subject":[],"published":{"date-parts":[[2022,5,21]]},"assertion":[{"value":"2022-07-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}