{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T21:21:18Z","timestamp":1782854478214,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,11,12]],"date-time":"2023-11-12T00:00:00Z","timestamp":1699747200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nd\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,11,12]]},"DOI":"10.1145\/3624062.3624135","type":"proceedings-article","created":{"date-parts":[[2023,11,10]],"date-time":"2023-11-10T13:53:39Z","timestamp":1699624419000},"page":"627-635","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":15,"title":["Towards Collaborative Continuous Benchmarking for HPC"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1904-9627","authenticated-orcid":false,"given":"Olga","family":"Pearce","sequence":"first","affiliation":[{"name":"Lawrence Livermore National Laboratory, United States of America and Texas A&M University, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6255-1308","authenticated-orcid":false,"given":"Alec","family":"Scott","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8472-4449","authenticated-orcid":false,"given":"Gregory","family":"Becker","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8930-5721","authenticated-orcid":false,"given":"Riyaz","family":"Haque","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2214-7447","authenticated-orcid":false,"given":"Nathan","family":"Hanford","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1458-8453","authenticated-orcid":false,"given":"Stephanie","family":"Brink","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3836-207X","authenticated-orcid":false,"given":"Doug","family":"Jacobsen","sequence":"additional","affiliation":[{"name":"Google, United States of America"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4104-3968","authenticated-orcid":false,"given":"Heidi","family":"Poxon","sequence":"additional","affiliation":[{"name":"Amazon, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5343-414X","authenticated-orcid":false,"given":"Jens","family":"Domke","sequence":"additional","affiliation":[{"name":"RIKEN, Japan"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7857-2805","authenticated-orcid":false,"given":"Todd","family":"Gamblin","sequence":"additional","affiliation":[{"name":"Lawrence Livermore National Laboratory, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,11,12]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3324989.3325719"},{"key":"e_1_3_2_2_2_1","volume-title":"Ubiquitous Performance Analysis","author":"Boehme David","unstructured":"David Boehme, Pascal Aschwanden, Olga Pearce, Kenneth Weiss, and Matthew LeGendre. 2021. Ubiquitous Performance Analysis. In High Performance Computing, Bradford\u00a0L. Chamberlain, Ana-Lucia Varbanescu, Hatem Ltaief, and Piotr Luszczek (Eds.). Springer International Publishing, Cham, 431\u2013449."},{"key":"e_1_3_2_2_3_1","volume-title":"Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis","author":"Boehme David","year":"2016","unstructured":"David Boehme, Todd Gamblin, David Beckingsale, Peer-Timo Bremer, Alfredo Gimenez, Matthew LeGendre, Olga Pearce, and Martin Schulz. 2016. Caliper: Performance Introspection for HPC Software Stacks. In Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis (Salt Lake City, Utah) (SC \u201916). IEEE Press, Article 47, 11\u00a0pages."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","unstructured":"Thomas Breuer Sebastian L\u00c3\u00bchrs Andreas Smolenko and Julia Wellmann. 2022. JUBE (Version 2.5.1); 2.5.1. https:\/\/doi.org\/10.5281\/ZENODO.7534373","DOI":"10.5281\/ZENODO.7534373"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3588195.3592989"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2503210.2503277"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/CANOPIEHPC51917.2020.00011"},{"key":"e_1_3_2_2_8_1","unstructured":"ECP. 2010. Jacamar-CI. https:\/\/gitlab.com\/ecp-ci\/jacamar-ci."},{"key":"e_1_3_2_2_9_1","unstructured":"Exascale Computing Project. 2018. ECP Proxy Apps Suite. https:\/\/proxyapps.exascaleproject.org\/ecp-proxy-apps-suite\/."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807623"},{"key":"e_1_3_2_2_11_1","unstructured":"Google. 2016. PerfKit Benchmarker. https:\/\/github.com\/GoogleCloudPlatform\/PerfKitBenchmarker."},{"key":"e_1_3_2_2_12_1","unstructured":"Google. 2023. Ramble. https:\/\/github.com\/GoogleCloudPlatform\/ramble."},{"key":"e_1_3_2_2_13_1","unstructured":"I. Bicking. 2011. pip: Package Install tool for Python. https:\/\/github.com\/pypa\/pip."},{"key":"e_1_3_2_2_14_1","unstructured":"I. Z. Schlueter. 2009. NPM. https:\/\/github.com\/npm\/npm."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3624062.3624132"},{"key":"e_1_3_2_2_16_1","unstructured":"LANL. 2014. Pavilion Framework. https:\/\/github.com\/hpc\/pavilion2."},{"key":"e_1_3_2_2_17_1","unstructured":"Carl Lerche Yehuda Katz and Andr\u00e9 Arko. 2010. Bundler. https:\/\/github.com\/rubygems\/bundler\/blob\/master\/LICENSE.md."},{"key":"e_1_3_2_2_18_1","unstructured":"LLNL. 2015. Spack. https:\/\/github.com\/spack\/spack."},{"key":"e_1_3_2_2_19_1","unstructured":"LLNL. 2017. Caliper. https:\/\/github.com\/llnl\/caliper."},{"key":"e_1_3_2_2_20_1","unstructured":"LLNL. 2019. Adiak. http:\/\/github.com\/LLNL\/adiak."},{"key":"e_1_3_2_2_21_1","volume-title":"AMG2023","author":"LLNL.","year":"2023","unstructured":"LLNL. 2023. AMG2023. https:\/\/github.com\/LLNL\/amg2023."},{"key":"e_1_3_2_2_22_1","unstructured":"LLNL. 2023. Benchpark. https:\/\/github.com\/LLNL\/benchpark."},{"key":"e_1_3_2_2_23_1","unstructured":"LLNL. 2023. Hubcast. https:\/\/github.com\/LLNL\/hubcast."},{"key":"e_1_3_2_2_24_1","unstructured":"LLNL. 2023. Thicket. https:\/\/github.com\/llnl\/thicket."},{"key":"e_1_3_2_2_25_1","unstructured":"Peter Mattson Christine Cheng Cody Coleman Greg Diamos Paulius Micikevicius David Patterson Hanlin Tang Gu-Yeon Wei Peter Bailis Victor Bittorf David Brooks Dehao Chen Debojyoti Dutta Udit Gupta Kim Hazelwood Andrew Hock Xinyuan Huang Atsushi Ike Bill Jia Daniel Kang David Kanter Naveen Kumar Jeffery Liao Guokai Ma Deepak Narayanan Tayo Oguntebi Gennady Pekhimenko Lillian Pentecost Vijay\u00a0Janapa Reddi Taylor Robie Tom\u00a0St. John Tsuguchika Tabaru Carole-Jean Wu Lingjie Xu Masafumi Yamazaki Cliff Young and Matei Zaharia. 2019. MLPerf Training Benchmark. arxiv:1910.01500\u00a0[cs.LG]"},{"key":"e_1_3_2_2_26_1","unstructured":"ML Commons. 2023. MLPerf. https:\/\/mlcommons.org\/en\/."},{"key":"e_1_3_2_2_27_1","volume-title":"Cargo: The Rust package manager. https:\/\/github.com\/rust-lang\/cargo.","year":"2014","unstructured":"Rust. 2014. Cargo: The Rust package manager. https:\/\/github.com\/rust-lang\/cargo."},{"key":"e_1_3_2_2_28_1","volume-title":"Python 3 Reference Manual. CreateSpace","author":"Van\u00a0Rossum Guido","unstructured":"Guido Van\u00a0Rossum and Fred\u00a0L. Drake. 2009. Python 3 Reference Manual. CreateSpace, Scotts Valley, CA."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/TRIDNT.2006.1649176"}],"event":{"name":"SC-W 2023: Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis","location":"Denver CO USA","acronym":"SC-W 2023"},"container-title":["Proceedings of the SC '23 Workshops of the International Conference on High Performance Computing, Network, Storage, and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3624062.3624135","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3624062.3624135","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,21]],"date-time":"2025-08-21T03:01:48Z","timestamp":1755745308000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3624062.3624135"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,11,12]]},"references-count":29,"alternative-id":["10.1145\/3624062.3624135","10.1145\/3624062"],"URL":"https:\/\/doi.org\/10.1145\/3624062.3624135","relation":{},"subject":[],"published":{"date-parts":[[2023,11,12]]},"assertion":[{"value":"2023-11-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}