{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,18]],"date-time":"2026-06-18T17:34:07Z","timestamp":1781804047563,"version":"3.54.5"},"reference-count":94,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2022]]},"DOI":"10.1109\/access.2022.3158675","type":"journal-article","created":{"date-parts":[[2022,3,10]],"date-time":"2022-03-10T20:25:06Z","timestamp":1646943906000},"page":"29207-29223","source":"Crossref","is-referenced-by-count":24,"title":["Reproducibility in Computing Research: An Empirical Study"],"prefix":"10.1109","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9927-5343","authenticated-orcid":false,"given":"Wullianallur","family":"Raghupathi","sequence":"first","affiliation":[{"name":"Gabelli School of Business, Fordham University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5082-3166","authenticated-orcid":false,"given":"Viju","family":"Raghupathi","sequence":"additional","affiliation":[{"name":"Koppelman School of Business, Brooklyn College, City University of New York, Brooklyn, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jie","family":"Ren","sequence":"additional","affiliation":[{"name":"Gabelli School of Business, Fordham University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1038\/530027a"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1802324115"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.cb310198"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3322790.3330595"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/2812803"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3097766.3097770"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/3456287.3465478"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1038\/505612a"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612465253"},{"issue":"26","key":"ref11","doi-asserted-by":"crossref","first-page":"452","DOI":"10.1038\/533452a","article-title":"Reproducibility crisis","volume":"533","author":"Baker","year":"2016","journal-title":"Nature"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0254090"},{"issue":"130","key":"ref13","article-title":"Scientists elusive goal: Reproducing study results","volume":"258","author":"Naik","year":"2011","journal-title":"Wall Street J."},{"key":"ref14","volume-title":"Trouble at the Lab","year":"2013"},{"issue":"4","key":"ref15","first-page":"76","article-title":"Lies, damned lies, and medical science","volume":"306","author":"Freedman","year":"2010","journal-title":"Atlantic"},{"key":"ref16","volume-title":"How Reliable are Psychology Studies? Atlantic","author":"Yong","year":"2015"},{"key":"ref17","volume-title":"How Reliable are Cancer Studies? Atlantic","author":"Yong","year":"2017"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac4716"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1214\/09-AOAS291"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.5860\/crl.79.1.2"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1111\/nyas.13325"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1038\/483531a"},{"issue":"8","key":"ref23","doi-asserted-by":"crossref","DOI":"10.1038\/nchem.2017","article-title":"Reproducibility","volume":"6","author":"Gibb","year":"2014","journal-title":"Nature Chem."},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1007\/s00704-015-1597-5"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1093\/cje\/bet075"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.3102\/0013189X14545513"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pbio.1002165"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.15252\/embr.201744876"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/2994031"},{"key":"ref30","article-title":"Reproducibility of data-oriented experiments in e-science (dagstuhl seminar 16041)","author":"Freire","year":"2016"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v40i4.5185"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11503"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v39i3.2816"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.17226\/25303"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1909046117"},{"key":"ref36","volume-title":"Reproducibility project: Psychology","author":"Aarts","year":"2015"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1161\/CIRCRESAHA.114.303819"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/3186266"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/3097766.3097767"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3344289"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1003285"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/3097766.3097768"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1109\/ETHICS.2016.7560050"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/2038642.2038650"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-38791-8_58"},{"issue":"10","key":"ref46","first-page":"1","article-title":"Encouraging reproducibility in scientific research of the internet","volume-title":"Dagstuhl Rep.","volume":"8","author":"Bajpai","year":"2019"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2017.12.029"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1126\/science.1213847"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.2172\/1481626"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0024357"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"ref52","volume-title":"Setting the default to reproducible. Reproducibility in Computational and Experimental Mathematics","author":"Stodden","year":"2017"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1190\/1.1822162"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1002\/9781118865064.ch9"},{"key":"ref55","volume-title":"Result and Artifact Review and Badging","year":"2016"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0067111"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/2213836.2213908"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/3020206"},{"key":"ref59","volume-title":"IEEE Reproducibility Practices Survey\u2014Summary of Findings","year":"2020"},{"key":"ref60","volume-title":"Study Shows Ensuring Reproducibility in Research Is Needed The IEEE Computer Society Suggests Improvements","author":"Goodrich","year":"2021"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/1096166.1096174"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2009.932122"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1708290115"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1126\/scitranslmed.aaf5027"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.15"},{"key":"ref66","article-title":"Rethinking experimental methods in computing (dagstuhl seminar 16111)","author":"Delling","year":"2016"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1145\/2658987"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459058"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1145\/3041027.3041028"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1145\/3214239.3214245"},{"key":"ref71","volume-title":"Researching Information Systems and Computing","author":"Oates","year":"2006"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1516179112"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611430953"},{"issue":"2","key":"ref74","first-page":"32","article-title":"Global measurements: Practice and experience (report on dagstuhl seminar# 16012)","volume-title":"ACM SIGCOMM Comput. Commun. Rev.","volume":"46","author":"Bajpai","year":"2016"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1145\/3364510.3364528"},{"key":"ref76","article-title":"Encouraging reproducibility in scientific research of the internet UCL-Universit\u00e9 Catholique de Louvain","author":"Bajpai","year":"2018"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-31744-1_52"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723875"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1201\/9781315373461-2"},{"key":"ref80","volume-title":"Docker-Based Solutions to Reproducibility in Science","year":"2015"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0188511"},{"key":"ref82","article-title":"A survey on reproducibility in parallel computing","author":"Hunold","year":"2015","journal-title":"arXiv:1511.04217"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1313476110"},{"key":"ref84","article-title":"On the state and importance of reproducible experimental research in parallel computing","author":"Hunold","year":"2013","journal-title":"arXiv:1308.3648"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1126\/science.359.6377.725"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723879"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-2544-7_5"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1807180116"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-33460-3_4"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1145\/381641.381656"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0250522"},{"issue":"5","key":"ref92","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1109\/MCG.2004.39","article-title":"Guest editors\u2019 introduction\u2013visual analytics","volume":"24","author":"Wong","year":"2004","journal-title":"IEEE Comput. Graph. Appl."},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2011.12.035"},{"key":"ref94","article-title":"Illuminating the path: Research and department agenda for visual analytics","author":"Thomas","year":"2005"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9668973\/09732944.pdf?arnumber=9732944","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,17]],"date-time":"2024-01-17T23:16:19Z","timestamp":1705533379000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9732944\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"references-count":94,"URL":"https:\/\/doi.org\/10.1109\/access.2022.3158675","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]}}}