{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,1]],"date-time":"2025-11-01T09:34:11Z","timestamp":1761989651190,"version":"3.28.0"},"reference-count":45,"publisher":"IEEE","license":[{"start":{"date-parts":[[2021,10,1]],"date-time":"2021-10-01T00:00:00Z","timestamp":1633046400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2021,10,1]],"date-time":"2021-10-01T00:00:00Z","timestamp":1633046400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021,10]]},"DOI":"10.1109\/issre52982.2021.00033","type":"proceedings-article","created":{"date-parts":[[2022,2,12]],"date-time":"2022-02-12T01:01:25Z","timestamp":1644627685000},"page":"220-230","source":"Crossref","is-referenced-by-count":10,"title":["Evaluating Natural Language Inference Models: A Metamorphic Testing Approach"],"prefix":"10.1109","author":[{"given":"Mingyue","family":"Jiang","sequence":"first","affiliation":[{"name":"Zhejiang Sci-Tech University,Hangzhou,China"}]},{"given":"Houzhen","family":"Bao","sequence":"additional","affiliation":[{"name":"Zhejiang Sci-Tech University,Hangzhou,China"}]},{"given":"Kaiyi","family":"Tu","sequence":"additional","affiliation":[{"name":"Zhejiang Sci-Tech University,Hangzhou,China"}]},{"given":"Xiao-Yi","family":"Zhang","sequence":"additional","affiliation":[{"name":"National Institute of Informatics,Tokyo,Japan"}]},{"given":"Zuohua","family":"Ding","sequence":"additional","affiliation":[{"name":"Zhejiang Sci-Tech University,Hangzhou,China"}]}],"member":"263","reference":[{"key":"ref39","article-title":"Metamorphic robustness testing: Exposing hidden defects in citation statistics and journal impact factors","author":"zhou","year":"2019","journal-title":"IEEE Transactions on Software Engineering"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2764464"},{"key":"ref33","article-title":"What if we simply swap the two text fragments? A straightforward yet effective way to test the robustness of methods to confounding signals in nature language inference tasks","volume":"abs 1809 2719","author":"wang","year":"2018","journal-title":"CoRR"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.3390\/math9070726"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR46437.2021.01663"},{"journal-title":"Biasfinder Metamorphic test generation to uncover bias for sentiment analysis systems","year":"2021","author":"asyrofi","key":"ref30"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1339"},{"journal-title":"Self-explaining structures improve NLP models","year":"2020","author":"sun","key":"ref36"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1465"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.661"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-2017"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2020.3003313"},{"key":"ref11","first-page":"67","article-title":"Collecting diverse natural language inference problems for sentence representation evaluation","author":"poliak","year":"0","journal-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing"},{"journal-title":"Beyond leaderboards A survey of methods for revealing weaknesses in natural language inference data and models","year":"2020","author":"schlegel","key":"ref12"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-2103"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-4804"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.665"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"ref17","first-page":"2340","article-title":"Stress test evaluation for natural language inference","author":"naik","year":"0","journal-title":"Proceedings of the 27th International Conference on Computational Linguistics"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2532875"},{"key":"ref19","first-page":"4: 1","article-title":"Metamorphic testing: A review of challenges and opportunities","volume":"51","author":"chen","year":"2018","journal-title":"ACM Computing Surveys"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/64"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/11736790_9"},{"key":"ref27","article-title":"Use of metamorphic relations as knowledge carriers to train deep neural networks","volume":"abs 2104 4718","author":"chen","year":"2021","journal-title":"CoRR"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D15-1075"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1441"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.442"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-1152"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380420"},{"key":"ref7","article-title":"Semantics-aware BERT for language understanding","author":"zhang","year":"2019","journal-title":"CoRR"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33016867"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1334"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.3115\/1599081.1599147"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2478001"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3387940.3391484"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2876433"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2012.08.008"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/QSIC.2009.26"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2020.2972266"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/ASWEC.2018.00021"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.23"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2016.04.002"}],"event":{"name":"2021 IEEE 32nd International Symposium on Software Reliability Engineering (ISSRE)","start":{"date-parts":[[2021,10,25]]},"location":"Wuhan, China","end":{"date-parts":[[2021,10,28]]}},"container-title":["2021 IEEE 32nd International Symposium on Software Reliability Engineering (ISSRE)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9700160\/9700163\/09700284.pdf?arnumber=9700284","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,7]],"date-time":"2022-12-07T01:49:13Z","timestamp":1670377753000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9700284\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,10]]},"references-count":45,"URL":"https:\/\/doi.org\/10.1109\/issre52982.2021.00033","relation":{},"subject":[],"published":{"date-parts":[[2021,10]]}}}