{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,10]],"date-time":"2026-04-10T10:07:08Z","timestamp":1775815628149,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,27]],"date-time":"2023-06-27T00:00:00Z","timestamp":1687824000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,27]]},"DOI":"10.1145\/3589806.3600041","type":"proceedings-article","created":{"date-parts":[[2023,6,28]],"date-time":"2023-06-28T20:09:22Z","timestamp":1687982962000},"page":"89-96","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["Does the Market of Citations Reward Reproducible Work?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9900-1972","authenticated-orcid":false,"given":"Edward","family":"Raff","sequence":"first","affiliation":[{"name":"Booz Allen Hamilton, USA and Department of Computer Science and Electrical Engineering, University of Maryland, Baltimore County, USA"}]}],"member":"320","published-online":{"date-parts":[[2023,6,28]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.23329"},{"key":"e_1_3_2_1_3_1","unstructured":"Xavier Bouthillier Pierre Delaunay Mirko Bronzi Assya Trofimov Brennan Nichyporuk Justin Szeto Naz Sepah Edward Raff Kanika Madan Vikram Voleti Samira\u00a0Ebrahimi Kahou Vincent Michalski Dmitriy Serdyuk Tal Arbel Chris Pal Ga\u00ebl Varoquaux and Pascal Vincent. 2021. Accounting for Variance in Machine Learning Benchmarks. In Machine Learning and Systems (MLSys). http:\/\/arxiv.org\/abs\/2103.03098"},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 36th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a097)","author":"Bouthillier Xavier","year":"2019","unstructured":"Xavier Bouthillier, C\u00e9sar Laurent, and Pascal Vincent. 2019. Unreproducible Research is Reproducible. In Proceedings of the 36th International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a097), Kamalika Chaudhuri and Ruslan Salakhutdinov (Eds.). PMLR, Long Beach, California, USA, 725\u2013734. http:\/\/proceedings.mlr.press\/v97\/bouthillier19a.html"},{"key":"e_1_3_2_1_5_1","volume-title":"REPRODUCIBLE RESEARCH WORKFLOW IN R FOR THE ANALYSIS OF PERSONALIZED HUMAN MICROBIOME DATA.Pacific Symposium on Biocomputing. Pacific Symposium on Biocomputing 21","author":"Callahan Benjamin","year":"2016","unstructured":"Benjamin Callahan, Diana Proctor, David Relman, Julia Fukuyama, and Susan Holmes. 2016. REPRODUCIBLE RESEARCH WORKFLOW IN R FOR THE ANALYSIS OF PERSONALIZED HUMAN MICROBIOME DATA.Pacific Symposium on Biocomputing. Pacific Symposium on Biocomputing 21 (2016), 183\u201394. http:\/\/www.ncbi.nlm.nih.gov\/pubmed\/26776185http:\/\/www.pubmedcentral.nih.gov\/articlerender.fcgi?artid=PMC4873301"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaf0918"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-018-0399-z"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1016\/0304-4076(90)90014-K"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1190\/1.1822162"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac4716"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1273496.1273526"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.15"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00074"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.7554\/eLife.67995"},{"key":"e_1_3_2_1_17_1","volume-title":"Reproducibility in ML Workshop, ICML\u201918","author":"Forde Jessica","year":"2018","unstructured":"Jessica Forde, Tim Head, Chris Holdgraf, Yuvi Panda, Fernando Perez, Gladys Nalvarte, Benjamin Ragan-kelley, and Erik Sundell. 2018. Reproducible Research Environments with repo2docker. In Reproducibility in ML Workshop, ICML\u201918."},{"key":"e_1_3_2_1_18_1","volume-title":"Reproducibility in ML Workshop, ICML\u201918","author":"Gardner Josh","year":"2018","unstructured":"Josh Gardner, Christopher Brooks, and Ryan\u00a0S Baker. 2018. Enabling End-To-End Machine Learning Replicability : A Case Study in Educational Data Mining. In Reproducibility in ML Workshop, ICML\u201918."},{"key":"e_1_3_2_1_19_1","first-page":"1593","article-title":"The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo","volume":"15","author":"Hoffman D","year":"2014","unstructured":"Matthew\u00a0D Hoffman and Andrew Gelman. 2014. The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo. Journal of Machine Learning Research 15, 47 (2014), 1593\u20131623. http:\/\/jmlr.org\/papers\/v15\/hoffman14a.html","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.359.6377.725"},{"key":"e_1_3_2_1_21_1","volume-title":"Positioning and Power in Academic Publishing: Players","author":"Kluyver Thomas","unstructured":"Thomas Kluyver, Benjamin Ragan-Kelley, Fernando P\u00e9rez, Brian Granger, Matthias Bussonnier, Jonathan Frederic, Kyle Kelley, Jessica Hamrick, Jason Grout, Sylvain Corlay, Paul Ivanov, Dami\u00e1n Avila, Safia Abdalla, Carol Willing, and Jupyter development team. 2016. Jupyter Notebooks - a publishing format for reproducible computational workflows. In Positioning and Power in Academic Publishing: Players, Agents and Agendas, Fernando Loizides and Birgit Scmidt (Eds.). IOS Press, 87\u201390. https:\/\/eprints.soton.ac.uk\/403913\/"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1162\/qss_a_00159"},{"key":"e_1_3_2_1_23_1","article-title":"The frequency distribution of scientific productivity","volume":"16","author":"Lotka J","year":"1926","unstructured":"Alfred\u00a0J Lotka. 1926. The frequency distribution of scientific productivity. Journal of the Washington Academy of Sciences 16, 12 (10 1926), 317\u2013323. http:\/\/www.jstor.org\/stable\/24529203","journal-title":"Journal of the Washington Academy of Sciences"},{"key":"e_1_3_2_1_24_1","volume-title":"Composable Effects for Flexible and Accelerated Probabilistic Programming in NumPyro. arXiv","author":"Phan Du","year":"2019","unstructured":"Du Phan, Neeraj Pradhan, and Martin Jankowiak. 2019. Composable Effects for Flexible and Accelerated Probabilistic Programming in NumPyro. arXiv (2019), 1\u201310. http:\/\/arxiv.org\/abs\/1912.11554"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.3389\/fninf.2017.00076"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuron.2018.11.030"},{"key":"e_1_3_2_1_27_1","first-page":"21","article-title":"Lotka\u2019s Law Revisited","volume":"30","author":"Potter William\u00a0Gray","year":"1981","unstructured":"William\u00a0Gray Potter. 1981. Lotka\u2019s Law Revisited. Library Trends 30 (1981), 21\u201339.","journal-title":"Library Trends"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.4630270505"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.149.3683.510"},{"key":"e_1_3_2_1_30_1","unstructured":"Edward Raff. 2019. A Step Toward Quantifying Independently Reproducible Machine Learning Research. In NeurIPS. http:\/\/arxiv.org\/abs\/1909.06674"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i1.16124"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1007\/s100510050359"},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the 28th International Conference on Neural Information Processing Systems -","volume":"2511","author":"Sculley D","year":"2015","unstructured":"D Sculley, Gary Holt, Daniel Golovin, Eugene Davydov, Todd Phillips, Dietmar Ebner, Vinay Chaudhary, Michael Young, Jean-Francois Crespo, and Dan Dennison. 2015. Hidden Technical Debt in Machine Learning Systems. In Proceedings of the 28th International Conference on Neural Information Processing Systems - Volume 2(NIPS\u201915). MIT Press, Cambridge, MA, USA, 2503\u20132511."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1126\/sciadv.abd1705"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/JRPROC.1957.278364"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.joi.2015.06.005"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412489"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.joi.2014.09.011"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1162\/qss_a_00128"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1905819116"},{"key":"e_1_3_2_1_41_1","volume-title":"Voodoo Correlations in Social Neuroscience. Perspectives on Psychological Science","author":"Vul Edward","year":"2008","unstructured":"Edward Vul, Christine Harris, Piotr Winkielman, and Harold Pashler. 2008. Voodoo Correlations in Social Neuroscience. Perspectives on Psychological Science (2008)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.joi.2009.03.010"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063757"}],"event":{"name":"ACM REP '23: 2023 ACM Conference on Reproducibility and Replicability","location":"Santa Cruz CA USA","acronym":"ACM REP '23","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2023 ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589806.3600041","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3589806.3600041","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:49:22Z","timestamp":1750182562000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589806.3600041"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,27]]},"references-count":43,"alternative-id":["10.1145\/3589806.3600041","10.1145\/3589806"],"URL":"https:\/\/doi.org\/10.1145\/3589806.3600041","relation":{},"subject":[],"published":{"date-parts":[[2023,6,27]]},"assertion":[{"value":"2023-06-28","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}