{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T18:07:53Z","timestamp":1776103673820,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":64,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,18]],"date-time":"2024-06-18T00:00:00Z","timestamp":1718668800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-sa\/4.0\/"}],"funder":[{"name":"NSERC"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,18]]},"DOI":"10.1145\/3641525.3663617","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T19:02:19Z","timestamp":1720724539000},"page":"1-19","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Computational Experiment Comprehension using Provenance Summarization"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0420-5519","authenticated-orcid":false,"given":"Nichole","family":"Boufford","sequence":"first","affiliation":[{"name":"University of British Columbia, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4024-239X","authenticated-orcid":false,"given":"Joseph","family":"Wonsil","sequence":"additional","affiliation":[{"name":"University of British Columbia, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2449-0844","authenticated-orcid":false,"given":"Adam","family":"Pocock","sequence":"additional","affiliation":[{"name":"Oracle Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-2877-401X","authenticated-orcid":false,"given":"Jack","family":"Sullivan","sequence":"additional","affiliation":[{"name":"Oracle Labs, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2165-4658","authenticated-orcid":false,"given":"Margo","family":"Seltzer","sequence":"additional","affiliation":[{"name":"University of British Columbia, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6876-1306","authenticated-orcid":false,"given":"Thomas","family":"Pasquier","sequence":"additional","affiliation":[{"name":"University of British Columbia, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n. d.]. Open AI Prompt Engineering Guide. https:\/\/platform.openai.com\/docs\/guides\/prompt-engineering. Accessed: 2023-11-22."},{"key":"e_1_3_2_1_2_1","unstructured":"(accessed January 23 2024). eBPF. online. https:\/\/ebpf.io\/. ."},{"key":"e_1_3_2_1_3_1","unstructured":"JJ Allaire Yihui Xie Christophe Dervieux Jonathan McPherson Javier Luraschi Kevin Ushey Aron Atkins Hadley Wickham Joe Cheng Winston Chang and Richard Iannone. 2023. rmarkdown: Dynamic Documents for R. https:\/\/github.com\/rstudio\/rmarkdown R package version 2.25."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/VISUAL.2005.1532788"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2013.155"},{"key":"e_1_3_2_1_7_1","unstructured":"Nichole Boufford. 2023. Thoth. https:\/\/github.com\/ubc-systopia\/thoth."},{"key":"e_1_3_2_1_8_1","volume-title":"Language Models are Few-Shot Learners. CoRR abs\/2005.14165","author":"Brown B.","year":"2020","unstructured":"Tom\u00a0B. Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel\u00a0M. Ziegler, Jeffrey Wu, Clemens Winter, Christopher Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. CoRR abs\/2005.14165 (2020). arXiv:2005.14165https:\/\/arxiv.org\/abs\/2005.14165"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2602649.2602651"},{"key":"e_1_3_2_1_10_1","volume-title":"Pro git","author":"Chacon Scott","unstructured":"Scott Chacon and Ben Straub. 2014. Pro git. Apress."},{"key":"e_1_3_2_1_11_1","volume-title":"BooookScore: A systematic exploration of book-length summarization in the era of LLMs. arXiv preprint arXiv:2310.00785","author":"Chang Yapei","year":"2023","unstructured":"Yapei Chang, Kyle Lo, Tanya Goyal, and Mohit Iyyer. 2023. BooookScore: A systematic exploration of book-length summarization in the era of LLMs. arXiv preprint arXiv:2310.00785 (2023)."},{"key":"e_1_3_2_1_12_1","unstructured":"Yupeng Chang Xu Wang Jindong Wang Yuan Wu Linyi Yang Kaijie Zhu Hao Chen Xiaoyuan Yi Cunxiang Wang Yidong Wang Wei Ye Yue Zhang Yi Chang Philip\u00a0S. Yu Qiang Yang and Xing Xie. 2023. A Survey on Evaluation of Large Language Models. arxiv:2307.03109\u00a0[cs.CL]"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2882903.2899401"},{"key":"e_1_3_2_1_14_1","volume-title":"Computational reproducibility via containers in social psychology. Meta-Psychology 3","author":"Clyburne-Sherin April","year":"2019","unstructured":"April Clyburne-Sherin, Xu Fei, and Seth\u00a0Ariel Green. 2019. Computational reproducibility via containers in social psychology. Meta-Psychology 3 (2019)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-76856-2_72"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-45848-4_57"},{"key":"e_1_3_2_1_17_1","first-page":"1","article-title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","volume":"23","author":"Fedus William","year":"2022","unstructured":"William Fedus, Barret Zoph, and Noam Shazeer. 2022. Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity. Journal of Machine Learning Research 23, 120 (2022), 1\u201339.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_18_1","unstructured":"Tanya Goyal Junyi\u00a0Jessy Li and Greg Durrett. 2023. News Summarization and Evaluation in the Era of GPT-3. arxiv:2209.12356\u00a0[cs.CL]"},{"key":"e_1_3_2_1_19_1","volume-title":"Burrito: Wrapping your lab notebook in computational infrastructure.","author":"Guo J","year":"2012","unstructured":"Philip\u00a0J Guo and Margo\u00a0I Seltzer. 2012. Burrito: Wrapping your lab notebook in computational infrastructure. (2012)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","unstructured":"Sandra\u00a0G. Hart and Lowell\u00a0E. Staveland. 1988. Development of NASA-TLX (Task Load Index): Results of Empirical and Theoretical Research. In Human Mental Workload Peter\u00a0A. Hancock and Najmedin Meshkati (Eds.). Advances in Psychology Vol.\u00a052. North-Holland 139\u2013183. https:\/\/doi.org\/10.1016\/S0166-4115(08)62386-9","DOI":"10.1016\/S0166-4115(08)62386-9"},{"key":"e_1_3_2_1_21_1","unstructured":"https:\/\/openai.com\/blog\/new-models-and-developer-products-announced-at devday. 2023. New models and developer products announced at DevDay. blog post."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.5555\/2737423.2737461"},{"key":"e_1_3_2_1_23_1","volume-title":"Rethinking Large Language Models in Mental Health Applications. arXiv preprint arXiv:2311.11267","author":"Ji Shaoxiong","year":"2023","unstructured":"Shaoxiong Ji, Tianlin Zhang, Kailai Yang, Sophia Ananiadou, and Erik Cambria. 2023. Rethinking Large Language Models in Mental Health Applications. arXiv preprint arXiv:2311.11267 (2023)."},{"key":"e_1_3_2_1_24_1","volume-title":"Positioning and Power in Academic Publishing: Players","author":"Kluyver Thomas","unstructured":"Thomas Kluyver, Benjamin Ragan-Kelley, Fernando P\u00e9rez, Brian Granger, Matthias Bussonnier, Jonathan Frederic, Kyle Kelley, Jessica Hamrick, Jason Grout, Sylvain Corlay, Paul Ivanov, Dami\u00e1n Avila, Safia Abdalla, Carol Willing, and Jupyter development team. 2016. Jupyter Notebooks - a publishing format for reproducible computational workflows. In Positioning and Power in Academic Publishing: Players, Agents and Agendas, Fernando Loizides and Birgit Scmidt (Eds.). IOS Press, Netherlands, 87\u201390. https:\/\/eprints.soton.ac.uk\/403913\/"},{"key":"e_1_3_2_1_25_1","volume-title":"Building Real-World Meeting Summarization Systems using Large Language Models: A Practical Perspective. arXiv preprint arXiv:2310","author":"Laskar Tahmid\u00a0Rahman","year":"2023","unstructured":"Md\u00a0Tahmid\u00a0Rahman Laskar, Xue-Yong Fu, Cheng Chen, and Shashi\u00a0Bhushan Tn. 2023. Building Real-World Meeting Summarization Systems using Large Language Models: A Practical Perspective. arXiv preprint arXiv:2310.19233 (2023)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-16462-5_36"},{"key":"e_1_3_2_1_27_1","volume-title":"A technique for the measurement of attitudes.Archives of psychology","author":"Likert Rensis","year":"1932","unstructured":"Rensis Likert. 1932. A technique for the measurement of attitudes.Archives of psychology (1932)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3472883.3486976"},{"key":"e_1_3_2_1_29_1","volume-title":"Lost in the middle: How language models use long contexts. arXiv preprint arXiv:2307.03172","author":"Liu F","year":"2023","unstructured":"Nelson\u00a0F Liu, Kevin Lin, John Hewitt, Ashwin Paranjape, Michele Bevilacqua, Fabio Petroni, and Percy Liang. 2023. Lost in the middle: How language models use long contexts. arXiv preprint arXiv:2307.03172 (2023)."},{"key":"e_1_3_2_1_30_1","unstructured":"Yucheng Low Ajit Banerjee and Rajat Arya. 2021. XetHub. https:\/\/about.xethub.com\/"},{"key":"e_1_3_2_1_31_1","volume-title":"Provenance Map Orbiter: Interactive Exploration of Large Provenance Graphs. In 3rd USENIX Workshop on the Theory and Practice of Provenance (TaPP 11)","author":"Macko Peter","year":"2011","unstructured":"Peter Macko and Margo Seltzer. 2011. Provenance Map Orbiter: Interactive Exploration of Large Provenance Graphs. In 3rd USENIX Workshop on the Theory and Practice of Provenance (TaPP 11). USENIX Association, Heraklion, Crete Greece. https:\/\/www.usenix.org\/conference\/tapp11\/provenance-map-orbiter-interactive-exploration-large-provenance-graphs"},{"key":"e_1_3_2_1_32_1","unstructured":"Andrew Mleczko Sebastian Schuberth Lars Schneider and Brian\u00a0M. Carlson. 2021. git-lfs. https:\/\/github.com\/git-lfs\/git-lfs"},{"key":"e_1_3_2_1_33_1","volume-title":"Provenance-Aware Storage Systems. USENIX (01","author":"Muniswamy-Reddy Kiran-Kumar","year":"2006","unstructured":"Kiran-Kumar Muniswamy-Reddy, David Holland, Uri Braun, and Margo Seltzer. 2006. Provenance-Aware Storage Systems. USENIX (01 2006), 43\u201356."},{"key":"e_1_3_2_1_34_1","volume-title":"Visualization Analysis and Design","author":"Munzner T.","unstructured":"T. Munzner. 2015. Visualization Analysis and Design. CRC Press. 210 pages. https:\/\/books.google.de\/books?id=NfkYCwAAQBAJ"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.17226\/25303"},{"key":"e_1_3_2_1_36_1","unstructured":"Neo4j. 2012. Neo4j. http:\/\/neo4j.org\/"},{"key":"e_1_3_2_1_37_1","unstructured":"OpenAI. 2023. ChatGPT. https:\/\/chat.openai.com\/chat"},{"key":"e_1_3_2_1_38_1","unstructured":"OpenAI. 2023. GPT-4 Technical Report. arxiv:2303.08774\u00a0[cs.CL]"},{"key":"e_1_3_2_1_39_1","unstructured":"Long Ouyang Jeff Wu Xu Jiang Diogo Almeida Carroll\u00a0L. Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray John Schulman Jacob Hilton Fraser Kelton Luke Miller Maddie Simens Amanda Askell Peter Welinder Paul Christiano Jan Leike and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. arxiv:2203.02155\u00a0[cs.CL]"},{"key":"e_1_3_2_1_40_1","volume-title":"Practical Whole-System Provenance Capture. In Symposium on Cloud Computing (SoCC\u201917)","author":"Pasquier Thomas","year":"2017","unstructured":"Thomas Pasquier, Xueyuan Han, Mark Goldstein, Thomas Moyer, David Eyers, Margo Seltzer, and Jean Bacon. 2017. Practical Whole-System Provenance Capture. In Symposium on Cloud Computing (SoCC\u201917). ACM, ACM."},{"key":"e_1_3_2_1_41_1","volume-title":"If these data could talk. Nature Scientific Data 4","author":"Pasquier Thomas","year":"2017","unstructured":"Thomas Pasquier, Matthew\u00a0K. Lau, Ana Trisovic, Emery\u00a0R. Boose, Ben Couturier, Merc\u00e8 Crosas, Aaron\u00a0M. Ellison, Valerie Gibson, Chris\u00a0R. Jones, and Margo Seltzer. 2017. If these data could talk. Nature Scientific Data 4 (2017). https:\/\/www.nature.com\/articles\/sdata2017114"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.14778\/3137765.3137789"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2019.00077"},{"key":"e_1_3_2_1_44_1","volume-title":"Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track.","author":"Ramakrishnan Chandrasekhar","year":"2023","unstructured":"Chandrasekhar Ramakrishnan, Michele Volpi, Fernando Perez-Cruz, Lilian Gasser, Firat Ozdemir, Patrick Paitz, Mohammad Alisafaee, Philipp Fischer, Ralf Grubenmann, Eliza\u00a0Jean Harris, 2023. Renku: a platform for sustainable data science. In Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track."},{"key":"e_1_3_2_1_45_1","volume-title":"9th USENIX Workshop on the Theory and Practice of Provenance (TaPP","author":"Schreiber Andreas","year":"2017","unstructured":"Andreas Schreiber and Regina Struminski. 2017. Visualizing Provenance using Comics. In 9th USENIX Workshop on the Theory and Practice of Provenance (TaPP 2017). USENIX Association, Seattle, WA. https:\/\/www.usenix.org\/conference\/tapp17\/workshop-program\/presentation\/schreiber"},{"key":"e_1_3_2_1_46_1","volume-title":"Scalable and Deployable Audit Data Collection System. In 2024 IEEE Symposium on Security and Privacy (SP). IEEE Computer Society, 87\u201387","author":"Sekar R","year":"2023","unstructured":"R Sekar, Hanke Kimm, and Rohit Aich. 2023. eAudit: A Fast, Scalable and Deployable Audit Data Collection System. In 2024 IEEE Symposium on Security and Privacy (SP). IEEE Computer Society, 87\u201387."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.simpa.2021.100068"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-emnlp.278"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-22295-5_19"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2022.23103"},{"key":"e_1_3_2_1_51_1","volume-title":"Evaluating large language models on medical evidence summarization. npj Digital Medicine 6, 1","author":"Tang Liyan","year":"2023","unstructured":"Liyan Tang, Zhaoyi Sun, Betina Idnay, Jordan\u00a0G Nestor, Ali Soroush, Pierre\u00a0A Elias, Ziyang Xu, Ying Ding, Greg Durrett, Justin\u00a0F Rousseau, 2023. Evaluating large language models on medical evidence summarization. npj Digital Medicine 6, 1 (2023), 158."},{"key":"e_1_3_2_1_52_1","volume-title":"Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, 2023. Llama 2: Open foundation and fine-tuned chat models. arXiv preprint arXiv:2307.09288 (2023)."},{"key":"e_1_3_2_1_53_1","volume-title":"Large Language Model Prompt Chaining for Long Legal Document Classification. arXiv preprint arXiv:2308.04138","author":"Trautmann Dietrich","year":"2023","unstructured":"Dietrich Trautmann. 2023. Large Language Model Prompt Chaining for Long Legal Document Classification. arXiv preprint arXiv:2308.04138 (2023)."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41597-022-01143-6"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1002\/spe.3202"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491101.3519729"},{"key":"e_1_3_2_1_57_1","volume-title":"Less is More for Long Document Summary Evaluation by LLMs. arXiv preprint arXiv:2309.07382","author":"Wu Yunshu","year":"2023","unstructured":"Yunshu Wu, Hayate Iso, Pouya Pezeshkpour, Nikita Bhutani, and Estevam Hruschka. 2023. Less is More for Long Document Summary Evaluation by LLMs. arXiv preprint arXiv:2309.07382 (2023)."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/2976749.2978378"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.emnlp-main.370"},{"key":"e_1_3_2_1_60_1","volume-title":"Scalability of network visualisation from a cognitive load perspective","author":"Yoghourdjian Vahan","year":"2020","unstructured":"Vahan Yoghourdjian, Yalong Yang, Tim Dwyer, Lee Lawrence, Michael Wybrow, and Kim Marriott. 2020. Scalability of network visualisation from a cognitive load perspective. IEEE transactions on visualization and computer graphics 27, 2 (2020), 1677\u20131687."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_2_1_62_1","volume-title":"SummIt: Iterative Text Summarization via ChatGPT. arXiv preprint arXiv:2305.14835","author":"Zhang Haopeng","year":"2023","unstructured":"Haopeng Zhang, Xiao Liu, and Jiawei Zhang. 2023. SummIt: Iterative Text Summarization via ChatGPT. arXiv preprint arXiv:2305.14835 (2023)."},{"key":"e_1_3_2_1_63_1","unstructured":"Tianyi Zhang Faisal Ladhak Esin Durmus Percy Liang Kathleen McKeown and Tatsunori\u00a0B. Hashimoto. 2023. Benchmarking Large Language Models for News Summarization. arxiv:2301.13848\u00a0[cs.CL]"},{"key":"e_1_3_2_1_64_1","unstructured":"Wayne\u00a0Xin Zhao Kun Zhou Junyi Li Tianyi Tang Xiaolei Wang Yupeng Hou Yingqian Min Beichen Zhang Junjie Zhang Zican Dong Yifan Du Chen Yang Yushuo Chen Zhipeng Chen Jinhao Jiang Ruiyang Ren Yifan Li Xinyu Tang Zikang Liu Peiyu Liu Jian-Yun Nie and Ji-Rong Wen. 2023. A Survey of Large Language Models. arxiv:2303.18223\u00a0[cs.CL]"}],"event":{"name":"ACM REP '24: ACM Conference on Reproducibility and Replicability","location":"Rennes France","acronym":"ACM REP '24","sponsor":["EIGREP Emerging Interest Group on Reproducibility and Replicability"]},"container-title":["Proceedings of the 2nd ACM Conference on Reproducibility and Replicability"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663617","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3641525.3663617","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:05:16Z","timestamp":1755914716000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3641525.3663617"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,18]]},"references-count":64,"alternative-id":["10.1145\/3641525.3663617","10.1145\/3641525"],"URL":"https:\/\/doi.org\/10.1145\/3641525.3663617","relation":{},"subject":[],"published":{"date-parts":[[2024,6,18]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}