{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,10]],"date-time":"2026-01-10T00:39:25Z","timestamp":1768005565312,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,29]]},"DOI":"10.1145\/3736731.3746150","type":"proceedings-article","created":{"date-parts":[[2025,10,21]],"date-time":"2025-10-21T11:58:19Z","timestamp":1761047899000},"page":"85-95","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["HPC Benchmarking: Repeat, Replicate, Reproduce"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1904-9627","authenticated-orcid":false,"given":"Olga","family":"Pearce","sequence":"first","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA and Texas A&M University, College Station, TX, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8472-4449","authenticated-orcid":false,"given":"Gregory","family":"Becker","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1458-8453","authenticated-orcid":false,"given":"Stephanie","family":"Brink","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2214-7447","authenticated-orcid":false,"given":"Nathan","family":"Hanford","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1453-5906","authenticated-orcid":false,"given":"Dewi","family":"Yokelson","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1549-0157","authenticated-orcid":false,"given":"August","family":"Knox","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0087-4301","authenticated-orcid":false,"given":"Barry","family":"Rountree","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, Livermore, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,10,21]]},"reference":[{"key":"e_1_3_3_2_2_2","unstructured":"AMD. 2025. Introduction to the HIP programming model. https:\/\/rocm.docs.amd.com\/projects\/HIP\/en\/latest\/understand\/programming_model.html."},{"key":"e_1_3_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641525.3663648"},{"key":"e_1_3_3_2_4_2","doi-asserted-by":"crossref","unstructured":"David Beckingsale Thomas R.\u00a0W. Scogland Jason Burmark Richard\u00a0D. Hornung Holger\u00a0E. Jones William Killian Adam\u00a0J. Kunen Olga Pearce Peter Robinson and Brian\u00a0S. Ryujin. 2019. RAJA: Portable Performance for Large-Scale Scientific Applications. 2019 IEEE\/ACM International Workshop on Performance Portability and Productivity in HPC (P3HPC) (2019) 71\u201381. https:\/\/api.semanticscholar.org\/CorpusID:209853208","DOI":"10.1109\/P3HPC49587.2019.00012"},{"key":"e_1_3_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC.2016.46"},{"key":"e_1_3_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641525.3663617"},{"key":"e_1_3_3_2_7_2","doi-asserted-by":"publisher","DOI":"10.1145\/3588195.3592989"},{"key":"e_1_3_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-14325-5_43"},{"key":"e_1_3_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.1145\/3641525.3663622"},{"key":"e_1_3_3_2_10_2","doi-asserted-by":"crossref","unstructured":"Ludovic Court\u00e8s. 2022. Reproducibility and Performance: Why Choose? Computing in Science & Engineering 24 3 (2022) 77\u201380. https:\/\/doi.org\/10.1109\/MCSE.2022.3165626","DOI":"10.1109\/MCSE.2022.3165626"},{"key":"e_1_3_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/2213836.2213908"},{"key":"e_1_3_3_2_12_2","doi-asserted-by":"crossref","unstructured":"Todd Gamblin Matthew LeGendre Michael\u00a0R. Collette Gregory\u00a0L. Lee Adam Moody Bronis\u00a0R. de Supinski and Scott Futral. 2015. The Spack Package Manager: Bringing Order to HPC Software Chaos(Supercomputing 2015 (SC\u201915)). Austin Texas US. https:\/\/tgamblin.github.io\/pubs\/spack-sc15.pdf","DOI":"10.1145\/2807591.2807623"},{"key":"e_1_3_3_2_13_2","unstructured":"Google. 2023. Ramble. https:\/\/github.com\/GoogleCloudPlatform\/ramble."},{"key":"e_1_3_3_2_14_2","series-title":"(SC \u201924)","volume-title":"Proceedings of the International Conference for High Performance Computing, Networking, Storage, and Analysis","author":"Herten Andreas","year":"2024","unstructured":"Andreas Herten, Sebastian Achilles, Damian Alvarez, Jayesh Badwaik, Eric Behle, Mathis Bode, Thomas Breuer, Daniel Caviedes-Voulli\u00e8me, Mehdi Cherti, Adel Dabah, Salem El\u00a0Sayed, Wolfgang Frings, Ana Gonzalez-Nicolas, Eric\u00a0B. Gregory, Kaveh\u00a0Haghighi Mood, Thorsten Hater, Jenia Jitsev, Chelsea\u00a0Maria John, Jan\u00a0H. Meinke, Catrin\u00a0I. Meyer, Pavel Mezentsev, Jan-Oliver Mirus, Stepan Nassyr, Carolin Penke, Manoel R\u00f6mmer, Ujjwal Sinha, Benedikt von St.\u00a0Vieth, Olaf Stein, Estela Suarez, Dennis Willsch, and Ilya Zhukov. 2024. Application-Driven Exascale: The JUPITER Benchmark Suite. In Proceedings of the International Conference for High Performance Computing, Networking, Storage, and Analysis (Atlanta, GA, USA) (SC \u201924). IEEE Press, Article 32, 45\u00a0pages. https:\/\/doi.org\/10.1109\/SC41406.2024.00038"},{"key":"e_1_3_3_2_15_2","volume-title":"Supercomputing","author":"Hoefler Torsten","year":"2015","unstructured":"Torsten Hoefler and Roberto Belli. 2015. Scientific benchmarking of parallel computing systems: twelve ways to tell the masses when reporting performance results. In Supercomputing."},{"key":"e_1_3_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3624062.3624132"},{"key":"e_1_3_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-44728-1_3"},{"key":"e_1_3_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3624062.3624133"},{"key":"e_1_3_3_2_19_2","doi-asserted-by":"crossref","unstructured":"Chris Lamb and Stefano Zacchiroli. 2022. Reproducible Builds: Increasing the Integrity of Software Supply Chains. IEEE Software 39 2 (2022) 62\u201370. https:\/\/doi.org\/10.1109\/MS.2021.3073045","DOI":"10.1109\/MS.2021.3073045"},{"key":"e_1_3_3_2_20_2","unstructured":"LANL. 2014. Pavilion Framework. https:\/\/github.com\/hpc\/pavilion2."},{"key":"e_1_3_3_2_21_2","unstructured":"LLNL. 2023. Benchpark. https:\/\/github.com\/LLNL\/benchpark."},{"key":"e_1_3_3_2_22_2","unstructured":"Sebastian L\u00fchrs Kay Thust Alexander Schnurpfeil Stephan Graf and Wolfgang Frings. 2015. JUBE \u2013 A Flexible Application- and Platform-Independent Environment for Benchmarking. ISC High Performance Frankfurt (Germany) 12 Jul 2015 - 16 Jul 2015. https:\/\/juser.fz-juelich.de\/record\/202788"},{"key":"e_1_3_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1145\/1508244.1508275"},{"key":"e_1_3_3_2_24_2","unstructured":"Non-for-profit effort. 2015. Reproducible Builds. https:\/\/reproducible-builds.org."},{"key":"e_1_3_3_2_25_2","unstructured":"NVIDIA. 2025. CUDA C++ Programming Guide. https:\/\/docs.nvidia.com\/cuda\/cuda-c-programming-guide\/."},{"key":"e_1_3_3_2_26_2","doi-asserted-by":"crossref","unstructured":"Olga Pearce Alec Scott Gregory Becker Riyaz Haque Nathan Hanford Stephanie Brink Doug Jacobsen Heidi Poxon Jens Domke and Todd Gamblin. 2023. Towards Collaborative Continuous Benchmarking for HPC. Association for Computing Machinery New York NY USA.","DOI":"10.1145\/3624062.3624135"},{"key":"e_1_3_3_2_27_2","doi-asserted-by":"crossref","unstructured":"Beth\u00a0A. Plale Tanu Malik and Line\u00a0C. Pouchard. 2021. Reproducibility Practice in High-Performance Computing: Community Survey Results. Computing in Science & Engineering 23 5 (2021) 55\u201360. https:\/\/doi.org\/10.1109\/MCSE.2021.3096678","DOI":"10.1109\/MCSE.2021.3096678"},{"key":"e_1_3_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTER.2017.121"},{"key":"e_1_3_3_2_29_2","doi-asserted-by":"crossref","unstructured":"Christian\u00a0R. Trott Damien Lebrun-Grandi\u00e9 Daniel Arndt Jan Ciesko Vinh\u00a0Q. Dang Nathan\u00a0D. Ellingwood Rahulkumar Gayatri Evan Harvey Daisy\u00a0S. Hollman Daniel Ibanez Nevin Liber Jonathan Madsen Jeff Miles David Poliakoff Amy Powell Sivasankaran Rajamanickam Mikael Simberg Daniel Sunderland Bruno Turcksin and Jeremiah\u00a0J. Wilke. 2022. Kokkos 3: Programming Model Extensions for the Exascale Era. IEEE Transactions on Parallel and Distributed Systems 33 (2022) 805\u2013817. https:\/\/api.semanticscholar.org\/CorpusID:238891079","DOI":"10.1109\/TPDS.2021.3097283"},{"key":"e_1_3_3_2_30_2","doi-asserted-by":"crossref","unstructured":"Jan Vitek and Tomas Kalibera. 2012. R3: repeatability reproducibility and rigor. ACM SIGPLAN Notices 47 (2012) 30\u201336. https:\/\/api.semanticscholar.org\/CorpusID:15629674","DOI":"10.1145\/2442776.2442781"}],"event":{"name":"ACM REP '25: ACM Conference on Reproducibility and Replicability","location":"Vancouver Canada","acronym":"ACM REP '25","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 3rd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3736731.3746150","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,9]],"date-time":"2026-01-09T18:04:33Z","timestamp":1767981873000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3736731.3746150"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,29]]},"references-count":29,"alternative-id":["10.1145\/3736731.3746150","10.1145\/3736731"],"URL":"https:\/\/doi.org\/10.1145\/3736731.3746150","relation":{},"subject":[],"published":{"date-parts":[[2025,7,29]]},"assertion":[{"value":"2025-10-21","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}