{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,6]],"date-time":"2026-08-06T00:20:55Z","timestamp":1785975655935,"version":"3.56.0"},"publisher-location":"New York, NY, USA","reference-count":65,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,17]],"date-time":"2022-05-17T00:00:00Z","timestamp":1652745600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,5,17]]},"DOI":"10.1145\/3524481.3527232","type":"proceedings-article","created":{"date-parts":[[2022,7,19]],"date-time":"2022-07-19T16:51:53Z","timestamp":1658249513000},"page":"40-51","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["A method and experiment to evaluate deep neural networks as test oracles for scientific software"],"prefix":"10.1145","author":[{"given":"Valdivino Alexandre","family":"de Santiago J\u00fanior","sequence":"first","affiliation":[{"name":"Coordena\u00e7\u00e3o de Pesquisa Aplicada e Desenvolvimento Tecnol\u00f3gico (COPDT), S\u00e3o Jos\u00e9 dos Campos, S\u00e3o Paulo, Brazil"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Fast Algorithms for Mining Association Rules. In 20th International Conference on Very Large Data Bases (VLDB). 487--499","author":"Agrawal R.","unstructured":"R. Agrawal and R. Srikant . 1994 . Fast Algorithms for Mining Association Rules. In 20th International Conference on Very Large Data Bases (VLDB). 487--499 . R. Agrawal and R. Srikant. 1994. Fast Algorithms for Mining Association Rules. In 20th International Conference on Very Large Data Bases (VLDB). 487--499."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.08.007"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110617"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE.2014.14"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40411-017-0043-z"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2019.06.012"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2020.110542"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/QSIC.2008.5"},{"key":"e_1_3_2_1_10_1","volume-title":"Alspector (Eds.)","volume":"6","author":"Bromley J.","year":"1993","unstructured":"J. Bromley , I. Guyon , Y. LeCun , E. Sackinger , and R. Shah . 1993. Signature Verification using a \"Siamese\" Time Delay Neural Network. In Advances in Neural Information Processing Systems, J. Cowan, G. Tesauro, and J . Alspector (Eds.) , Vol. 6 . Morgan-Kaufmann. https:\/\/proceedings.neurips.cc\/paper\/ 1993 \/file\/288cc0ff022877bd3df94bc9360b9c5d-Paper.pdf J. Bromley, I. Guyon, Y. LeCun, E. Sackinger, and R. Shah. 1993. Signature Verification using a \"Siamese\" Time Delay Neural Network. In Advances in Neural Information Processing Systems, J. Cowan, G. Tesauro, and J. Alspector (Eds.), Vol. 6. Morgan-Kaufmann. https:\/\/proceedings.neurips.cc\/paper\/1993\/file\/288cc0ff022877bd3df94bc9360b9c5d-Paper.pdf"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/TCE.2020.2986049"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.envsoft.2013.03.002"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-020-19393-6"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.463"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2011.06.009"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-010-9135-7"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 27th International Conference on Neural Information Processing Systems -","volume":"2","author":"Goodfellow I. J.","unstructured":"I. J. Goodfellow , J. Pouget-Abadie , M. Mirza , B. Xu , D. Warde-Farley , S. Ozair , A. Courville , and Y. Bengio . 2014. Generative Adversarial Nets . In Proceedings of the 27th International Conference on Neural Information Processing Systems - Volume 2 (Montreal, Canada) (NIPS'14). MIT Press, Cambridge, MA, USA, 2672--2680. I. J. Goodfellow, J. Pouget-Abadie, M. Mirza, B. Xu, D. Warde-Farley, S. Ozair, A. Courville, and Y. Bengio. 2014. Generative Adversarial Nets. In Proceedings of the 27th International Conference on Neural Information Processing Systems - Volume 2 (Montreal, Canada) (NIPS'14). MIT Press, Cambridge, MA, USA, 2672--2680."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2011.73"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2015.7102580"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.07.011"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the 31st International Conference on Neural Information Processing Systems","author":"Heusel M.","unstructured":"M. Heusel , H. Ramsauer , T. Unterthiner , B. Nessler , and S. Hochreiter . 2017. GANs Trained by a Two Time-Scale Update Rule Converge to a Local Nash Equilibrium . In Proceedings of the 31st International Conference on Neural Information Processing Systems ( Long Beach, California, USA) (NIPS 17). Curran Associates Inc., Red Hook, NY, USA, 6629--6640. M. Heusel, H. Ramsauer, T. Unterthiner, B. Nessler, and S. Hochreiter. 2017. GANs Trained by a Two Time-Scale Update Rule Converge to a Local Nash Equilibrium. In Proceedings of the 31st International Conference on Neural Information Processing Systems (Long Beach, California, USA) (NIPS 17). Curran Associates Inc., Red Hook, NY, USA, 6629--6640."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.243"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/IBCAST.2016.7429896"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.71"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3128473.3128484"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2014.05.006"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2017.06.023"},{"key":"e_1_3_2_1_30_1","volume-title":"Weinberger (Eds.)","volume":"25","author":"Krizhevsky A.","year":"2012","unstructured":"A. Krizhevsky , I. Sutskever , and G. E. Hinton . 2012. ImageNet Classification with Deep Convolutional Neural Networks. In Advances in Neural Information Processing Systems, F. Pereira, C. J. C. Burges, L. Bottou, and K. Q . Weinberger (Eds.) , Vol. 25 . Curran Associates, Inc. https:\/\/proceedings.neurips.cc\/paper\/ 2012 \/file\/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf A. Krizhevsky, I. Sutskever, and G. E. Hinton. 2012. ImageNet Classification with Deep Convolutional Neural Networks. In Advances in Neural Information Processing Systems, F. Pereira, C. J. C. Burges, L. Bottou, and K. Q. Weinberger (Eds.), Vol. 25. Curran Associates, Inc. https:\/\/proceedings.neurips.cc\/paper\/2012\/file\/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2013.01.008"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.biopha.2020.110255"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.atmosenv.2019.04.002"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324884.3416547"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236024.3236082"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2013.44"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2015.7102586"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2016.93"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.iot.2020.100227"},{"key":"e_1_3_2_1_41_1","volume-title":"MLOps: From Model-centric to Data-centric AI. Available from: https:\/\/bit.ly\/31L7lx8. Access on","author":"Ng A.","year":"2022","unstructured":"A. Ng . 2021. MLOps: From Model-centric to Data-centric AI. Available from: https:\/\/bit.ly\/31L7lx8. Access on : March 4, 2022 . A. Ng. 2021. MLOps: From Model-centric to Data-centric AI. Available from: https:\/\/bit.ly\/31L7lx8. Access on: March 4, 2022."},{"key":"e_1_3_2_1_42_1","volume-title":"Proceedings of the International Conference on Software Engineering and Knowledge Engineering, SEKE","author":"Oliveira R. A. P.","year":"2014","unstructured":"R. A. P. Oliveira , A. M. Memon , V. N. Gil , F. L. S. Nunes , and M. Delamaro . 2014. An extensible framework to implement test oracles for non-testable programs . Proceedings of the International Conference on Software Engineering and Knowledge Engineering, SEKE January ( 2014 ), 199--204. https:\/\/www.scopus.com\/inward\/record.uri?eid=2-s2.0-84935074236&partnerID=40&md5=778fb3296d6f16acf77a7e70702d9a9d R. A. P. Oliveira, A. M. Memon, V. N. Gil, F. L. S. Nunes, and M. Delamaro. 2014. An extensible framework to implement test oracles for non-testable programs. Proceedings of the International Conference on Software Engineering and Knowledge Engineering, SEKE January (2014), 199--204. https:\/\/www.scopus.com\/inward\/record.uri?eid=2-s2.0-84935074236&partnerID=40&md5=778fb3296d6f16acf77a7e70702d9a9d"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397354"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3361566"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2011.6126544"},{"key":"e_1_3_2_1_46_1","volume-title":"Test Oracle based on CNN (TOrC): Supplemental Material. Available from: https:\/\/github.com\/vsantjr\/TOrC. Access on","author":"Santiago J\u00fanior V. A.","year":"2022","unstructured":"V. A. Santiago J\u00fanior . 2022. Test Oracle based on CNN (TOrC): Supplemental Material. Available from: https:\/\/github.com\/vsantjr\/TOrC. Access on : March 4, 2022 . V. A. Santiago J\u00fanior. 2022. Test Oracle based on CNN (TOrC): Supplemental Material. Available from: https:\/\/github.com\/vsantjr\/TOrC. Access on: March 4, 2022."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3266003.3266004"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/DICTA.2014.7008093"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-019-0197-0"},{"key":"e_1_3_2_1_50_1","first-page":"6","article-title":"Evaluating the Effectiveness of Metamorphic Testing on Edge Detection Programs","volume":"4","author":"Sim K. Y.","year":"2013","unstructured":"K. Y. Sim , D. M. L. Wong , and T. Y. Hii . 2013 . Evaluating the Effectiveness of Metamorphic Testing on Edge Detection Programs . In International Journal of Innovation, Management and Technology , Vol. 4. 6 -- 10 . K. Y. Sim, D. M. L. Wong, and T. Y. Hii. 2013. Evaluating the Effectiveness of Metamorphic Testing on Edge Detection Programs. In International Journal of Innovation, Management and Technology, Vol. 4. 6--10.","journal-title":"International Journal of Innovation, Management and Technology"},{"key":"e_1_3_2_1_51_1","unstructured":"K. Simonyan and A. Zisserman. 2015. Very Deep Convolutional Networks for Large-Scale Image Recognition. In 3rd International Conference on Learning Representations ICLR 2015 San Diego CA USA May 7-9 2015 Conference Track Proceedings Yoshua Bengio and Yann LeCun (Eds.). http:\/\/arxiv.org\/abs\/1409.1556  K. Simonyan and A. Zisserman. 2015. Very Deep Convolutional Networks for Large-Scale Image Recognition. In 3rd International Conference on Learning Representations ICLR 2015 San Diego CA USA May 7-9 2015 Conference Track Proceedings Yoshua Bengio and Yann LeCun (Eds.). http:\/\/arxiv.org\/abs\/1409.1556"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.envres.2015.10.010"},{"key":"e_1_3_2_1_54_1","volume-title":"Google says sorry for racist auto-tag in photo app. Available from: https:\/\/bit.ly\/3y8OHeJ. Access on","year":"2022","unstructured":"The Guardian. 2015. Google says sorry for racist auto-tag in photo app. Available from: https:\/\/bit.ly\/3y8OHeJ. Access on : March 4, 2022 . The Guardian. 2015. Google says sorry for racist auto-tag in photo app. Available from: https:\/\/bit.ly\/3y8OHeJ. Access on: March 4, 2022."},{"key":"e_1_3_2_1_55_1","volume-title":"Tesla Says Autopilot Makes Its Cars Safer. Crash Victims Say It Kills. Available from: https:\/\/nyti.ms\/3EER6Ag. Access on","year":"2022","unstructured":"The New York Times. 2021. Tesla Says Autopilot Makes Its Cars Safer. Crash Victims Say It Kills. Available from: https:\/\/nyti.ms\/3EER6Ag. Access on : March 4, 2022 . The New York Times. 2021. Tesla Says Autopilot Makes Its Cars Safer. Crash Victims Say It Kills. Available from: https:\/\/nyti.ms\/3EER6Ag. Access on: March 4, 2022."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380400"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jairtraman.2020.101822"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/DASC.2011.39"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIP.2003.819861"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-016-0043-6"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.634"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1162\/neco_a_01199"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.5244\/C30.87"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"}],"event":{"name":"AST '22: IEEE\/ACM 3rd International Conference on Automation of Software Test","location":"Pittsburgh Pennsylvania","acronym":"AST '22","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE TCSC IEEE Technical Committee on Scalable Computing"]},"container-title":["Proceedings of the 3rd ACM\/IEEE International Conference on Automation of Software Test"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3524481.3527232","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3524481.3527232","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:50:59Z","timestamp":1750182659000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3524481.3527232"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,17]]},"references-count":65,"alternative-id":["10.1145\/3524481.3527232","10.1145\/3524481"],"URL":"https:\/\/doi.org\/10.1145\/3524481.3527232","relation":{},"subject":[],"published":{"date-parts":[[2022,5,17]]},"assertion":[{"value":"2022-07-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}