{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,5]],"date-time":"2026-02-05T10:12:54Z","timestamp":1770286374875,"version":"3.49.0"},"reference-count":69,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"4","license":[{"start":{"date-parts":[[2020,12,1]],"date-time":"2020-12-01T00:00:00Z","timestamp":1606780800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"National Key R&D Program of China","award":["2018YFB1003901"],"award-info":[{"award-number":["2018YFB1003901"]}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61572375"],"award-info":[{"award-number":["61572375"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61972289"],"award-info":[{"award-number":["61972289"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61832009"],"award-info":[{"award-number":["61832009"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61772263"],"award-info":[{"award-number":["61772263"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Rel."],"published-print":{"date-parts":[[2020,12]]},"DOI":"10.1109\/tr.2020.2972266","type":"journal-article","created":{"date-parts":[[2020,3,13]],"date-time":"2020-03-13T20:05:56Z","timestamp":1584129956000},"page":"1293-1322","source":"Crossref","is-referenced-by-count":58,"title":["METTLE: A METamorphic Testing Approach to Assessing and Validating Unsupervised Machine Learning Systems"],"prefix":"10.1109","volume":"69","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2828-7165","authenticated-orcid":false,"given":"Xiaoyuan","family":"Xie","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0029-0912","authenticated-orcid":false,"given":"Zhiyi","family":"Zhang","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3578-0994","authenticated-orcid":false,"given":"Tsong Yueh","family":"Chen","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7300-9215","authenticated-orcid":false,"given":"Yang","family":"Liu","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2840-2418","authenticated-orcid":false,"given":"Pak-Lok","family":"Poon","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7743-1296","authenticated-orcid":false,"given":"Baowen","family":"Xu","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1016\/0031-3203(87)90081-1"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-9-42"},{"key":"ref33","first-page":"109","article-title":"A comparison of document clustering techniques","author":"steinbach","year":"0","journal-title":"Proc KDD Workshop Text Mining"},{"key":"ref32","author":"han","year":"2011","journal-title":"Data Mining Concepts and Techniques"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"ref37","first-page":"289","article-title":"Resampling methods for unsupervised learning from sample data","author":"m\u00f6ller","year":"2009","journal-title":"Machine Learning"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1016\/j.csda.2006.11.025"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1162\/089976604773717621"},{"key":"ref34","author":"kaufman","year":"2009","journal-title":"Finding Groups in Data An Introduction to Cluster Analysis"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2876433"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/DASFAA.2001.916368"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/TFUZZ.1997.554463"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2532875"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2017.2"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2764464"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2018.2850315"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.01.013"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/3241979"},{"key":"ref67","article-title":"Quality engineering in the new: A vision and R&D update from Accenture labs and Accenture testing services","year":"2018"},{"key":"ref68","article-title":"GraphicsFuzz launches testing solution for graphics drivers","year":"2018","journal-title":"Imperial Innovations"},{"key":"ref69","article-title":"Google acquires GraphicsFuzz, a service that tests android graphics drivers","author":"lardinois","year":"2018","journal-title":"TechCrunch com"},{"key":"ref2","first-page":"39","article-title":"A review of fraud detection techniques: Credit card","volume":"45","author":"chaudhry","year":"2012","journal-title":"Int J Comput Appl"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.2307\/3151680"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1201\/b19706"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2015.07.037"},{"key":"ref21","article-title":"Metamorphic testing: A new approach for generating next test cases","author":"chen","year":"1998"},{"key":"ref24","first-page":"867","article-title":"Properties of machine learning applications for use in metamorphic testing","author":"murphy","year":"0","journal-title":"Proc 20th Int Conf Softw Eng Knowl Eng"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2934848"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2478001"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1137\/1026034"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcss.2004.10.006"},{"key":"ref59","first-page":"1","article-title":"Looking for an MR? Try METWiki today","author":"xie","year":"0","journal-title":"Proceedings of the 1st International Workshop on Metamorphic Testing"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00108"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238202"},{"key":"ref56","article-title":"Machine learning testing: Survey, landscapes and horizons","author":"zhang","year":"2019"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/267580.267590"},{"key":"ref54","first-page":"3133","article-title":"Do we need hundreds of classifiers to solve real world classification problems","volume":"15","author":"fernandez-delgado","year":"2014","journal-title":"J Mach Learn Res"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/1143844.1143865"},{"key":"ref52","first-page":"2825","article-title":"Scikit-Learn: Machine learning in python","volume":"12","author":"pedregosa","year":"2011","journal-title":"J Mach Learn Res"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1142\/9789813235533_0018"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2011.2134230"},{"key":"ref40","article-title":"Towards dataset dynamics: Change frequency of linked open data sources","author":"umbrich","year":"0","journal-title":"Proceedings of 3rd International Workshop on Linked Data on the Web"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1145\/1014052.1014112"},{"key":"ref13","article-title":"What is clustering and why is it hard","author":"williams","year":"2015"},{"key":"ref14","first-page":"65","article-title":"Clustering: Science or art","author":"von luxburg","year":"0","journal-title":"International Conference on Unsupervised and Transfer Learning Workshop"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/3143561"},{"key":"ref16","first-page":"27","article-title":"Internal versus external cluster validation indexes","volume":"5","author":"rend\u00f3n","year":"2011","journal-title":"Int J Comput Commun"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9066-8"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2010.35"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2012.07.021"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/956750.956801"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2004.68"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2015.2461676"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2003.1250972"},{"key":"ref8","first-page":"478","article-title":"Unsupervised deep embedding for clustering analysis","author":"xie","year":"0","journal-title":"Proc 33rd Int Conf Int Conf Mach Learn"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TETC.2014.2330519"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-71887-3"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2017.06.053"},{"key":"ref46","first-page":"226","article-title":"A density-based algorithm for discovering clusters in large spatial databases with noise","author":"ester","year":"0","journal-title":"Proc Int'l Conf Knowledge Discovery and Data Mining"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.3233\/IDA-2007-11602"},{"key":"ref48","first-page":"727","article-title":"X-means: Extending k-means with efficient estimation of the number of clusters","volume":"1","author":"pelleg","year":"0","journal-title":"Proc 17th Int Conf Mach Learn"},{"key":"ref47","author":"witten","year":"2017","journal-title":"Data Mining Practical Machine Learning Tools and Techniques"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/1083231.1083240"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.2307\/2346830"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-29044-2"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-00427-8_2"}],"container-title":["IEEE Transactions on Reliability"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/24\/9274445\/09036058.pdf?arnumber=9036058","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T16:03:31Z","timestamp":1642003411000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9036058\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,12]]},"references-count":69,"journal-issue":{"issue":"4"},"URL":"https:\/\/doi.org\/10.1109\/tr.2020.2972266","relation":{},"ISSN":["0018-9529","1558-1721"],"issn-type":[{"value":"0018-9529","type":"print"},{"value":"1558-1721","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,12]]}}}