{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,25]],"date-time":"2026-02-25T22:39:59Z","timestamp":1772059199998,"version":"3.50.1"},"reference-count":83,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"6","license":[{"start":{"date-parts":[[2021,6,1]],"date-time":"2021-06-01T00:00:00Z","timestamp":1622505600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/OAPA.html"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["LP160101691"],"award-info":[{"award-number":["LP160101691"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Australian Government Research Training Program scholarship"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2021,6,1]]},"DOI":"10.1109\/tse.2019.2915065","type":"journal-article","created":{"date-parts":[[2019,5,8]],"date-time":"2019-05-08T00:51:23Z","timestamp":1557276683000},"page":"1164-1183","source":"Crossref","is-referenced-by-count":27,"title":["Metamorphic Robustness Testing: Exposing Hidden Defects in Citation Statistics and Journal Impact Factors"],"prefix":"10.1109","volume":"47","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1769-9367","authenticated-orcid":false,"given":"Zhi Quan","family":"Zhou","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0460-8377","authenticated-orcid":false,"given":"T. H.","family":"Tse","sequence":"additional","affiliation":[]},{"given":"Matt","family":"Witheridge","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1145\/96267.96279"},{"key":"ref72","first-page":"126","article-title":"The impact of title length and punctuation marks on article citations","volume":"62","author":"fumania","year":"2015","journal-title":"Annals of Library and Information Studies"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.6061\/clinics\/2012(05)17"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1007\/s11192-011-0412-z"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/bxm043"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/TSA.2015.13"},{"key":"ref74","author":"takanen","year":"2018","journal-title":"Fuzzing for Software Security Testing and Quality Assurance"},{"key":"ref39","year":"0"},{"key":"ref75","article-title":"Metamorphic testing techniques to detect defects in applications without test oracles","author":"murphy","year":"2010"},{"key":"ref38","year":"0"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2017.6"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238187"},{"key":"ref33","author":"ghezzi","year":"2002","journal-title":"Fundamentals of Software Engineering"},{"key":"ref32","year":"2011","journal-title":"Systems and software engineering – Systems and software Quality Requirements and Evaluation (SQuaRE) – System and software quality models"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2010.11.920"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1145\/2896971.2896978"},{"key":"ref37","author":"cleveland","year":"2013","journal-title":"Introduction to Indexing and Abstracting"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1002\/asi.21451"},{"key":"ref35","first-page":"35","article-title":"Fuzz testing for software assurance","volume":"28","author":"okun","year":"2015","journal-title":"Crosstalk J of Defense Software Eng"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2014.273"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1111\/j.1740-9713.2006.00202.x"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2778898"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1002\/asi.20653"},{"key":"ref63","author":"field","year":"2017","journal-title":"Discovering statistics using IBM SPSS statistics North American edition"},{"key":"ref28","year":"0"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1093\/ps\/86.4.603"},{"key":"ref27","article-title":"Quality engineering in the new: A vision and R&D update from Accenture Labs and Accenture Testing Services","year":"2018"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1007\/s11432-016-5544-6"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.3923\/jse.2018.1.11"},{"key":"ref29","year":"0"},{"key":"ref67","first-page":"100","article-title":"Differential testing for software","volume":"10","author":"mckeeman","year":"1998","journal-title":"Digit Tech J"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.6017\/ital.v36i2.9718"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.3325\/cmj.2010.51.165"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2014.2372785"},{"key":"ref1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/B978-0-12-800160-8.00001-2","article-title":"Automated test oracles: A survey","volume":"95","author":"pezz\u00e8","year":"2014","journal-title":"Advances in Computers"},{"key":"ref20","article-title":"Finding compiler bugs by removing dead code.","author":"regehr","year":"2014"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3133917"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2016.176"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/3193977.3193984"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.24251\/HICSS.2018.713"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213858"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/MET.2017..1"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pbio.1000242"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1001\/jama.295.1.90"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.5539\/ass.v9n5p18"},{"key":"ref58","article-title":"Citations per faculty","year":"0"},{"key":"ref57","article-title":"Discover the data behind the Times Higher Education World University Rankings","author":"hanafi","year":"2018"},{"key":"ref56","article-title":"Data from: The advantage of short paper titles","author":"letchford","year":"2015"},{"key":"ref55","doi-asserted-by":"crossref","DOI":"10.1038\/nature.2015.18246","article-title":"Papers with shorter titles get more citations: Intriguing correlation mined from 140,000 papers","author":"deng","year":"2015","journal-title":"Nature"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1126\/science.aad1669"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1098\/rsos.150266"},{"key":"ref52","doi-asserted-by":"crossref","first-page":"16569","DOI":"10.1073\/pnas.0507655102","article-title":"An index to quantify an individual’s scientific research output","volume":"102","author":"hirsch","year":"2005","journal-title":"Proc Natl Acad Sci United States America"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/3193977.3193979"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180220"},{"key":"ref40","year":"0"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TR.2018.2847247"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3143561"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3241979"},{"key":"ref15","article-title":"Automated functional testing of web search engines in the absence of an oracle","author":"zhou","year":"2007"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1002\/SERIES1345"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.437"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2876433"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2478001"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2764464"},{"key":"ref83","author":"lehmann","year":"2004","journal-title":"Elements of Large-Sample Theory"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/2666356.2594334"},{"key":"ref80","first-page":"1","article-title":"Metamorphic testing: Challenges ahead (keynote speech)","author":"segura","year":"2018","journal-title":"Proc IEEE\/ACM 3rd Int Workshop Metamorphic Testing"},{"key":"ref4","article-title":"Metamorphic testing: A new approach for generating next test cases","author":"chen","year":"1998"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1007\/s11219-017-9392-4"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.348"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1016\/S0950-5849(02)00129-5"},{"key":"ref8","article-title":"Message from the workshop chairs","author":"kanewala","year":"2016","journal-title":"Proc IEEE\/ACM 1st Int Workshop Metamorphic Testing"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2016.2532875"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1038\/4441003a"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/3183440.3183468"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.23"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/CMPSAC.2001.960614"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1214\/09-STS285"},{"key":"ref47","author":"wilsdon","year":"2016","journal-title":"The Metric Tide Report of the Independent Review of the Role of Metrics in Research Assessment and Management"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1136\/emj.2004.022103"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2009.20"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/QSIC.2006.45"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1136\/emj.2006.044206"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/32\/9453223\/08708940.pdf?arnumber=8708940","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,12,17]],"date-time":"2021-12-17T19:58:37Z","timestamp":1639771117000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8708940\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6,1]]},"references-count":83,"journal-issue":{"issue":"6"},"URL":"https:\/\/doi.org\/10.1109\/tse.2019.2915065","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,6,1]]}}}