{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,12]],"date-time":"2026-03-12T15:27:11Z","timestamp":1773329231966,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":57,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,27]],"date-time":"2023-06-27T00:00:00Z","timestamp":1687824000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,27]]},"DOI":"10.1145\/3589806.3600032","type":"proceedings-article","created":{"date-parts":[[2023,6,28]],"date-time":"2023-06-28T20:09:22Z","timestamp":1687982962000},"page":"62-73","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["KheOps: Cost-effective Repeatability, Reproducibility, and Replicability of Edge-to-Cloud Experiments"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1175-8426","authenticated-orcid":false,"given":"Daniel","family":"Rosendo","sequence":"first","affiliation":[{"name":"Computer Science Department, Inria, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5251-5466","authenticated-orcid":false,"given":"Kate","family":"Keahey","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Argonne National Laboratory, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3111-6308","authenticated-orcid":false,"given":"Alexandru","family":"Costan","sequence":"additional","affiliation":[{"name":"Computer Science Department, Inria, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9063-0334","authenticated-orcid":false,"given":"Matthieu","family":"Simonin","sequence":"additional","affiliation":[{"name":"Computer Science Department, Inria, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6506-7538","authenticated-orcid":false,"given":"Patrick","family":"Valduriez","sequence":"additional","affiliation":[{"name":"Computer Science Department, Inria, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6525-3736","authenticated-orcid":false,"given":"Gabriel","family":"Antoniu","sequence":"additional","affiliation":[{"name":"Computer Science Department, Inria, France"}]}],"member":"320","published-online":{"date-parts":[[2023,6,28]]},"reference":[{"key":"e_1_3_2_3_1_1","unstructured":"[n. d.]. Artifact Review and Badging Version 1.1.https:\/\/www.acm.org\/publications\/policies\/artifact-review-and-badging-current"},{"key":"e_1_3_2_3_2_1","volume-title":"d.]. What is Docker Hub?Retrieved","year":"2023","unstructured":"[n. d.]. What is Docker Hub?Retrieved Jun 1, 2023 from https:\/\/www.docker.com\/products\/docker-hub\/"},{"key":"e_1_3_2_3_3_1","unstructured":"2018. Dool (Dstat) monitoring.Retrieved Jan 14 2023 from https:\/\/github.com\/scottchiefbaker\/dool"},{"key":"e_1_3_2_3_4_1","unstructured":"2018. GitHub.Retrieved Jan 14 2023 from https:\/\/github.com\/"},{"key":"e_1_3_2_3_5_1","unstructured":"2018. Zenodo.Retrieved Jan 14 2023 from https:\/\/zenodo.org\/"},{"key":"e_1_3_2_3_6_1","volume-title":"E2Clab source code.Retrieved","year":"2023","unstructured":"2019. E2Clab source code.Retrieved Jan 14, 2023 from https:\/\/gitlab.inria.fr\/E2Clab\/e2clab"},{"key":"e_1_3_2_3_7_1","unstructured":"2023. AI Hub.Retrieved Jan 14 2023 from https:\/\/aihub.cloud.google.com\/"},{"key":"e_1_3_2_3_8_1","unstructured":"2023. Apache Zeppelin.Retrieved Jan 15 2023 from https:\/\/zeppelin.apache.org\/"},{"key":"e_1_3_2_3_9_1","volume-title":"Code Ocean Explore: Open Science Library.Retrieved","year":"2023","unstructured":"2023. Code Ocean Explore: Open Science Library.Retrieved Jan 19, 2023 from https:\/\/codeocean.com\/explore"},{"key":"e_1_3_2_3_10_1","volume-title":"Colab: Cloud Storage from the command line.Retrieved","year":"2023","unstructured":"2023. Colab: Cloud Storage from the command line.Retrieved Jan 18, 2023 from https:\/\/cloud.google.com\/storage\/docs\/gsutil"},{"key":"e_1_3_2_3_11_1","volume-title":"Colab: Google Spreadsheets.Retrieved","year":"2023","unstructured":"2023. Colab: Google Spreadsheets.Retrieved Jan 18, 2023 from https:\/\/github.com\/burnash\/gspread#more-examples"},{"key":"e_1_3_2_3_12_1","volume-title":"Compute skylake cluster at CHI@TACC.Retrieved","year":"2023","unstructured":"2023. Compute skylake cluster at CHI@TACC.Retrieved Feb 16, 2023 from https:\/\/www.chameleoncloud.org\/hardware\/node\/sites\/tacc\/clusters\/chameleon\/nodes\/0b0bceb9-14bf-423e-890f-3ef187511d71\/"},{"key":"e_1_3_2_3_13_1","volume-title":"Dahu cluster.Retrieved","year":"2023","unstructured":"2023. Dahu cluster.Retrieved Feb 16, 2023 from https:\/\/www.grid5000.fr\/w\/Grenoble:Hardware#dahu"},{"key":"e_1_3_2_3_14_1","unstructured":"2023. Docker.Retrieved Jan 18 2023 from https:\/\/www.docker.com\/"},{"key":"e_1_3_2_3_15_1","volume-title":"E2Clab User Defined Services.Retrieved","year":"2023","unstructured":"2023. E2Clab User Defined Services.Retrieved Feb 8, 2023 from https:\/\/gitlab.inria.fr\/E2Clab\/user-defined-services"},{"key":"e_1_3_2_3_16_1","volume-title":"Experiment artifacts.Retrieved","year":"2023","unstructured":"2023. Experiment artifacts.Retrieved Feb 8, 2023 from https:\/\/www.chameleoncloud.org\/experiment\/share\/347adbf3-7c14-4834-b802-b45fdd0d9564"},{"key":"e_1_3_2_3_17_1","volume-title":"Experiment results.Retrieved","year":"2023","unstructured":"2023. Experiment results.Retrieved Jan 14, 2023 from https:\/\/gitlab.inria.fr\/E2Clab\/Paper-Artifacts"},{"key":"e_1_3_2_3_18_1","unstructured":"2023. Google Colab.Retrieved Jan 17 2023 from https:\/\/colab.research.google.com\/"},{"key":"e_1_3_2_3_19_1","volume-title":"Google Colab: Frequently Asked Questions.Retrieved","year":"2023","unstructured":"2023. Google Colab: Frequently Asked Questions.Retrieved Jan 18, 2023 from https:\/\/research.google.com\/colaboratory\/faq.html"},{"key":"e_1_3_2_3_20_1","volume-title":"Retrieved","year":"2023","unstructured":"2023. Google Colab vs Kaggle. Retrieved Jan 20, 2023 from https:\/\/datasciencenotebook.org\/compare\/colab\/kaggle"},{"key":"e_1_3_2_3_21_1","volume-title":"Kaggle community.Retrieved","year":"2023","unstructured":"2023. Kaggle community.Retrieved Jan 19, 2023 from https:\/\/www.kaggle.com\/"},{"key":"e_1_3_2_3_22_1","volume-title":"Kaggle datasets.Retrieved","year":"2023","unstructured":"2023. Kaggle datasets.Retrieved Jan 20, 2023 from https:\/\/www.kaggle.com\/datasets"},{"key":"e_1_3_2_3_23_1","volume-title":"MQTT: The Standard for IoT Messaging.Retrieved","year":"2023","unstructured":"2023. MQTT: The Standard for IoT Messaging.Retrieved Feb 16, 2023 from https:\/\/mqtt.org\/"},{"key":"e_1_3_2_3_24_1","volume-title":"Python zlib.Retrieved","year":"2023","unstructured":"2023. Python zlib.Retrieved Feb 16, 2023 from https:\/\/docs.python.org\/3\/library\/zlib.html"},{"key":"e_1_3_2_3_25_1","volume-title":"Raspberry Pi 3 Model B.Retrieved","year":"2023","unstructured":"2023. Raspberry Pi 3 Model B.Retrieved Feb 16, 2023 from https:\/\/www.iot-lab.info\/docs\/boards\/raspberry-pi-3\/"},{"key":"e_1_3_2_3_26_1","unstructured":"2023. Raspberry Pi 4.Retrieved Feb 16 2023 from https:\/\/chameleoncloud.org\/experiment\/chiedge\/hardware-info\/"},{"key":"e_1_3_2_3_27_1","volume-title":"SC: The largest Reproducibility Laboratory.Retrieved","year":"2023","unstructured":"2023. SC: The largest Reproducibility Laboratory.Retrieved Feb 8, 2023 from https:\/\/www.chameleoncloud.org\/blog\/2023\/02\/20\/sc-the-largest-reproducibility-laboratory\/"},{"key":"e_1_3_2_3_28_1","volume-title":"Trovi: Practical Open Reproducibility.Retrieved","year":"2023","unstructured":"2023. Trovi: Practical Open Reproducibility.Retrieved Jan 20, 2023 from https:\/\/chameleoncloud.gitbook.io\/trovi\/"},{"key":"e_1_3_2_3_29_1","unstructured":"2023. Yocto Project.Retrieved Jan 14 2023 from https:\/\/www.yoctoproject.org\/"},{"key":"e_1_3_2_3_30_1","volume-title":"Zooniverse dataset.Retrieved","year":"2023","unstructured":"2023. Zooniverse dataset.Retrieved Feb 16, 2023 from https:\/\/www.zooniverse.org\/organizations\/meredithspalmer\/snapshot-safari"},{"key":"e_1_3_2_3_31_1","volume-title":"Tensorflow: A system for large-scale machine learning. In 12th { USENIX} symposium on operating systems design and implementation ({ OSDI} 16). 265\u2013283.","author":"Abadi Mart\u00edn","year":"2016","unstructured":"Mart\u00edn Abadi, Paul Barham, Jianmin Chen, Zhifeng Chen, Andy Davis, Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Geoffrey Irving, Michael Isard, 2016. Tensorflow: A system for large-scale machine learning. In 12th { USENIX} symposium on operating systems design and implementation ({ OSDI} 16). 265\u2013283."},{"key":"e_1_3_2_3_32_1","volume-title":"FIT IoT-LAB: A large scale open experimental IoT testbed. In 2015 IEEE 2nd World Forum on Internet of Things (WF-IoT)","author":"Adjih Cedric","unstructured":"Cedric Adjih, Emmanuel Baccelli, Eric Fleury, Gaetan Harter, Nathalie Mitton, Thomas Noel, Roger Pissard-Gibollet, Frederic Saint-Marcel, Guillaume Schreiner, Julien Vandaele, 2015. FIT IoT-LAB: A large scale open experimental IoT testbed. In 2015 IEEE 2nd World Forum on Internet of Things (WF-IoT). IEEE, 459\u2013464."},{"key":"e_1_3_2_3_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/CloudCom.2019.00032"},{"key":"e_1_3_2_3_34_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2017.3971172"},{"key":"e_1_3_2_3_35_1","doi-asserted-by":"publisher","DOI":"10.1177\/1094342006070078"},{"key":"e_1_3_2_3_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2021.3111159"},{"key":"e_1_3_2_3_37_1","volume-title":"Computational reproducibility via containers in psychology. Meta-psychology 3","author":"Clyburne-Sherin April","year":"2019","unstructured":"April Clyburne-Sherin, Xu Fei, and Seth\u00a0Ariel Green. 2019. Computational reproducibility via containers in psychology. Meta-psychology 3 (2019)."},{"key":"e_1_3_2_3_38_1","doi-asserted-by":"publisher","DOI":"10.1083\/jcb.200611141"},{"key":"e_1_3_2_3_39_1","unstructured":"ETP4HPC. April 29 2020. ETP4HPC Strategic Research Agenda. https:\/\/www.etp4hpc.eu\/sra.html."},{"key":"e_1_3_2_3_40_1","volume-title":"On reproducible AI: Towards reproducible research, open science, and digital scholarship in AI publications. AI magazine 39, 3","author":"Gundersen Odd\u00a0Erik","year":"2018","unstructured":"Odd\u00a0Erik Gundersen, Yolanda Gil, and David\u00a0W Aha. 2018. On reproducible AI: Towards reproducible research, open science, and digital scholarship in AI publications. AI magazine 39, 3 (2018), 56\u201368."},{"key":"e_1_3_2_3_41_1","volume-title":"Anna Goldenberg","author":"Haibe-Kains Benjamin","year":"2020","unstructured":"Benjamin Haibe-Kains, George\u00a0Alexandru Adam, Ahmed Hosny, Farnoosh Khodakarami, Massive Analysis Quality Control (MAQC) Society\u00a0Board of Directors Shraddha Thakkar 35 Kusko Rebecca 36 Sansone Susanna-Assunta 37 Tong Weida 35 Wolfinger Russ D. 38 Mason Christopher E. 39 Jones Wendell 40 Dopazo Joaquin 41 Furlanello Cesare\u00a042, Levi Waldron, Bo Wang, Chris McIntosh, Anna Goldenberg, Anshul Kundaje, 2020. Transparency and reproducibility in artificial intelligence. Nature 586, 7829 (2020), E14\u2013E16."},{"key":"e_1_3_2_3_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2020.3013361"},{"key":"e_1_3_2_3_43_1","unstructured":"Kate Keahey Jason Anderson Michael Sherman Zhuo Zhen Mark Powers Isabel Brunkan and Adam Cooper. 2021. Chameleon@Edge Community Workshop Report."},{"key":"e_1_3_2_3_44_1","volume-title":"2020 USENIX Annual Technical Conference (USENIX ATC 20)","author":"Keahey Kate","year":"2020","unstructured":"Kate Keahey, Jason Anderson, Zhuo Zhen, Pierre Riteau, Paul Ruth, Dan Stanzione, Mert Cevik, Jacob Colleran, Haryadi\u00a0S Gunawi, Cody Hammock, 2020. Lessons learned from the chameleon testbed. In 2020 USENIX Annual Technical Conference (USENIX ATC 20). 219\u2013233."},{"key":"e_1_3_2_3_45_1","unstructured":"Thomas Kluyver Benjamin Ragan-Kelley Fernando P\u00e9rez Brian\u00a0E Granger Matthias Bussonnier Jonathan Frederic Kyle Kelley Jessica\u00a0B Hamrick Jason Grout Sylvain Corlay 2016. Jupyter Notebooks-a publishing format for reproducible computational workflows. Vol.\u00a02016."},{"key":"e_1_3_2_3_46_1","article-title":"Learning from reproducing computational results: introducing three principles and the Reproduction Package","volume":"379","author":"Krafczyk S","year":"2021","unstructured":"Matthew\u00a0S Krafczyk, A Shi, Adhithya Bhaskar, D Marinov, and Victoria Stodden. 2021. Learning from reproducing computational results: introducing three principles and the Reproduction Package. Philosophical Transactions of the Royal Society A 379, 2197 (2021), 20200069.","journal-title":"Philosophical Transactions of the Royal Society A"},{"key":"e_1_3_2_3_47_1","volume-title":"Encyclopedia of database systems. Vol.\u00a06","author":"Liu Ling","unstructured":"Ling Liu and M\u00a0Tamer \u00d6zsu. 2009. Encyclopedia of database systems. Vol.\u00a06. Springer."},{"key":"e_1_3_2_3_48_1","volume-title":"Reproducibility and replicability in science","author":"Engineering National Academies\u00a0of Sciences, Medicine, 2019.","unstructured":"Engineering National Academies\u00a0of Sciences, Medicine, 2019. Reproducibility and replicability in science. National Academies Press."},{"key":"e_1_3_2_3_49_1","volume-title":"Pytorch: An imperative style, high-performance deep learning library. Advances in neural information processing systems 32","author":"Paszke Adam","year":"2019","unstructured":"Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, 2019. Pytorch: An imperative style, high-performance deep learning library. Advances in neural information processing systems 32 (2019), 8026\u20138037."},{"key":"e_1_3_2_3_50_1","volume-title":"Scikit-learn: Machine learning in Python. the Journal of machine Learning research 12","author":"Pedregosa Fabian","year":"2011","unstructured":"Fabian Pedregosa, Ga\u00ebl Varoquaux, Alexandre Gramfort, Vincent Michel, Bertrand Thirion, Olivier Grisel, Mathieu Blondel, Peter Prettenhofer, Ron Weiss, Vincent Dubourg, 2011. Scikit-learn: Machine learning in Python. the Journal of machine Learning research 12 (2011), 2825\u20132830."},{"key":"e_1_3_2_3_51_1","doi-asserted-by":"publisher","DOI":"10.1109\/Cluster48925.2021.00043"},{"key":"e_1_3_2_3_52_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2022.04.004"},{"key":"e_1_3_2_3_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/CLUSTER49012.2020.00028"},{"key":"e_1_3_2_3_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2019.05.011"},{"key":"e_1_3_2_3_55_1","volume-title":"Enhancing reproducibility for computational methods. Science 354, 6317","author":"Stodden Victoria","year":"2016","unstructured":"Victoria Stodden, Marcia McNutt, David\u00a0H Bailey, Ewa Deelman, Yolanda Gil, Brooks Hanson, Michael\u00a0A Heroux, John\u00a0PA Ioannidis, and Michela Taufer. 2016. Enhancing reproducibility for computational methods. Science 354, 6317 (2016), 1240\u20131241."},{"key":"e_1_3_2_3_56_1","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"e_1_3_2_3_57_1","volume-title":"The FAIR Guiding Principles for scientific data management and stewardship. Scientific data 3, 1","author":"Wilkinson D","year":"2016","unstructured":"Mark\u00a0D Wilkinson, Michel Dumontier, IJsbrand\u00a0Jan Aalbersberg, Gabrielle Appleton, Myles Axton, Arie Baak, Niklas Blomberg, Jan-Willem Boiten, Luiz\u00a0Bonino da Silva\u00a0Santos, Philip\u00a0E Bourne, 2016. The FAIR Guiding Principles for scientific data management and stewardship. Scientific data 3, 1 (2016), 1\u20139."}],"event":{"name":"ACM REP '23: 2023 ACM Conference on Reproducibility and Replicability","location":"Santa Cruz CA USA","acronym":"ACM REP '23","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2023 ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589806.3600032","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3589806.3600032","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:49:22Z","timestamp":1750182562000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3589806.3600032"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,27]]},"references-count":57,"alternative-id":["10.1145\/3589806.3600032","10.1145\/3589806"],"URL":"https:\/\/doi.org\/10.1145\/3589806.3600032","relation":{},"subject":[],"published":{"date-parts":[[2023,6,27]]},"assertion":[{"value":"2023-06-28","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}