{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T18:10:29Z","timestamp":1776103829068,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,19]],"date-time":"2023-04-19T00:00:00Z","timestamp":1681862400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,19]]},"DOI":"10.1145\/3544548.3581099","type":"proceedings-article","created":{"date-parts":[[2023,4,20]],"date-time":"2023-04-20T04:28:44Z","timestamp":1681964924000},"page":"1-19","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Understanding and Supporting Debugging Workflows in Multiverse Analysis"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4343-1578","authenticated-orcid":false,"given":"Ken","family":"Gu","sequence":"first","affiliation":[{"name":"Allen School of Computer Science & Engineering, University of Washington, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4050-4284","authenticated-orcid":false,"given":"Eunice","family":"Jun","sequence":"additional","affiliation":[{"name":"Allen School of Computer Science & Engineering, University of Washington, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4793-2289","authenticated-orcid":false,"given":"Tim","family":"Althoff","sequence":"additional","affiliation":[{"name":"Allen School of Computer Science & Engineering, University of Washington, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,4,19]]},"reference":[{"key":"e_1_3_3_3_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/3-540-29297-7_16"},{"key":"e_1_3_3_3_2_1","unstructured":"2022. difflib \u2014 Helpers for computing deltas \u2014 Python 3.10.7 documentation. https:\/\/docs.python.org\/3\/library\/difflib.html"},{"key":"e_1_3_3_3_3_1","unstructured":"2022. Introduction \u2014 statsmodels. https:\/\/www.statsmodels.org\/"},{"key":"e_1_3_3_3_4_1","unstructured":"2022. pandas - Python Data Analysis Library. https:\/\/pandas.pydata.org\/"},{"key":"e_1_3_3_3_5_1","unstructured":"2022. SPSS Software. https:\/\/www.ibm.com\/analytics\/spss-statistics-software"},{"key":"e_1_3_3_3_6_1","unstructured":"2022. Stack Overflow - Where Developers Learn Share and Build Careers. https:\/\/stackoverflow.com\/"},{"key":"e_1_3_3_3_7_1","unstructured":"2022. Statistical software for data science | Stata. https:\/\/www.stata.com\/"},{"key":"e_1_3_3_3_8_1","unstructured":"Abdulaziz Alaboudi and Thomas\u00a0D. LaToza. 2021. An Exploratory Study of Debugging Episodes. http:\/\/arxiv.org\/abs\/2105.02162 arXiv:2105.02162 [cs]."},{"key":"e_1_3_3_3_9_1","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"e_1_3_3_3_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180175"},{"key":"e_1_3_3_3_11_1","unstructured":"Chris van\u00a0den Berg. 2020. String Grouper. https:\/\/github.com\/Bergvca\/string_grouper"},{"key":"e_1_3_3_3_12_1","volume-title":"Testing Object-Oriented Systems: Models, Patterns, and Tools","author":"Binder V.","unstructured":"Robert\u00a0V. Binder. 1999. Testing Object-Oriented Systems: Models, Patterns, and Tools. Addison-Wesley Longman Publishing Co., Inc., USA."},{"key":"e_1_3_3_3_13_1","volume-title":"Into the Multiverse: Methods for Studying Developmental Neuroscience. Ph","author":"Bloom Paul\u00a0Alexander","year":"1958","unstructured":"Paul\u00a0Alexander Bloom. 2022. Into the Multiverse: Methods for Studying Developmental Neuroscience. Ph.D. Columbia University, United States \u2013 New York. https:\/\/www.proquest.com\/docview\/2656195811\/abstract\/673E811D3C8B4B71PQ\/1ISBN: 9798426817869."},{"key":"e_1_3_3_3_14_1","unstructured":"Zack Bobbitt. 2022. Python Guides. https:\/\/www.statology.org\/python-guides\/"},{"key":"e_1_3_3_3_15_1","unstructured":"Zack Bobbitt. 2022. R Guides. https:\/\/www.statology.org\/r-guides\/"},{"key":"e_1_3_3_3_16_1","doi-asserted-by":"publisher","DOI":"10.1176\/appi.ajp.2018.18070881"},{"key":"e_1_3_3_3_17_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2203150119"},{"key":"e_1_3_3_3_18_1","doi-asserted-by":"publisher","DOI":"10.1177\/1948550618775108"},{"key":"e_1_3_3_3_19_1","doi-asserted-by":"publisher","DOI":"10.1037\/pspp0000186"},{"key":"e_1_3_3_3_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300295"},{"key":"e_1_3_3_3_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642982"},{"key":"e_1_3_3_3_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642937.2642982"},{"key":"e_1_3_3_3_23_1","doi-asserted-by":"publisher","unstructured":"Kiran Gadhave Jochen G\u00f6rtler Zach\u00a0Tyler Cutler Carolina Nobre Oliver Deussen Miriah Meyer Jeff Phillips and Alexander Lex. 2020. Predicting Intent Behind Selections in Scatterplot Visualizations. https:\/\/doi.org\/10.31219\/osf.io\/mq2rk","DOI":"10.31219\/osf.io"},{"key":"e_1_3_3_3_24_1","unstructured":"Joachim Gassen. 2022. Researcher Degrees of Freedom Analysis. https:\/\/joachim-gassen.github.io\/rdfanalysis\/index.html"},{"key":"e_1_3_3_3_25_1","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.14443"},{"key":"e_1_3_3_3_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3332165.3347940"},{"key":"e_1_3_3_3_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501888"},{"key":"e_1_3_3_3_28_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1402786111"},{"key":"e_1_3_3_3_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300432"},{"key":"e_1_3_3_3_30_1","doi-asserted-by":"publisher","DOI":"10.1177\/0956797619838763"},{"key":"e_1_3_3_3_31_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4684-2001-2_9"},{"key":"e_1_3_3_3_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025626"},{"key":"e_1_3_3_3_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2018.8506576"},{"key":"e_1_3_3_3_34_1","unstructured":"Ruben Kleiman Mike Brayshaw Marc Eisenstadt and Marc Eisenstadt. 1993. Tales of Debugging from The Front Lines."},{"key":"e_1_3_3_3_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2006.116"},{"key":"e_1_3_3_3_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300754"},{"key":"e_1_3_3_3_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376533"},{"key":"e_1_3_3_3_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2020.3028985"},{"key":"e_1_3_3_3_39_1","unstructured":"Microsoft. 2022. Monaco Editor. https:\/\/microsoft.github.io\/monaco-editor\/"},{"key":"e_1_3_3_3_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPC.2015.12"},{"key":"e_1_3_3_3_41_1","doi-asserted-by":"publisher","unstructured":"Guiomar Niso Rotem Botvinik-Nezer Stefan Appelhoff Alejandro De\u00a0La Vega Oscar Esteban Joset\u00a0A. Etzel Karolina Finc Melanie Ganz Remi Gau Yaroslav\u00a0O. Halchenko Peer Herholz Agah Karakuzu David Keator Camille Maumet Christopher\u00a0J. Markiewicz Dr\u00a0Cyril Pernet Franco Pestilli Nazek Queder Tina Schmitt Weronika S\u00f3jka Adina\u00a0Svenja Wagner Kirstie Whitaker and Jochem Rieger. 2022. Open and reproducible neuroimaging: from study inception to publication. https:\/\/doi.org\/10.31219\/osf.io\/pu5vb","DOI":"10.31219\/osf.io"},{"key":"e_1_3_3_3_42_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac4716"},{"key":"e_1_3_3_3_43_1","unstructured":"Mark Otto and Jacob Thornton. 2022. Bootstrap. https:\/\/getbootstrap.com\/"},{"key":"e_1_3_3_3_44_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2015.05.029"},{"key":"e_1_3_3_3_45_1","doi-asserted-by":"publisher","DOI":"10.1177\/1367006918763132"},{"key":"e_1_3_3_3_46_1","doi-asserted-by":"publisher","DOI":"10.1177\/0956797619830649"},{"key":"e_1_3_3_3_47_1","unstructured":"Armin Ronacher. 2022. Welcome to Flask \u2014 Flask Documentation (2.2.x). https:\/\/flask.palletsprojects.com\/en\/2.2.x\/"},{"key":"e_1_3_3_3_49_1","volume-title":"Programming Puzzles. Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1 (Dec. 2021","author":"Schuster Tal","year":"2021","unstructured":"Tal Schuster, Ashwin Kalyan, Alex Polozov, and Adam Kalai. 2021. Programming Puzzles. Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks 1 (Dec. 2021). https:\/\/datasets-benchmarks-proceedings.neurips.cc\/paper\/2021\/hash\/3988c7f88ebcb58c6ce932b957b6f332-Abstract-round1.html"},{"key":"e_1_3_3_3_50_1","doi-asserted-by":"publisher","unstructured":"Martin Schweinsberg Michael Feldman Nicola Staub Olmo\u00a0R. van\u00a0den Akker Robbie\u00a0C.M. van Aert Marcel\u00a0A.L.M. van Assen Yang Liu Tim Althoff Jeffrey Heer Alex Kale Zainab Mohamed Hashem Amireh Vaishali Venkatesh\u00a0Prasad Abraham Bernstein Emily Robinson Kaisa Snellman S. Amy\u00a0Sommer Sarah\u00a0M.G. Otner David Robinson Nikhil Madan Raphael Silberzahn Pavel Goldstein Warren Tierney Toshio Murase Benjamin Mandl Domenico Viganola Carolin Strobl Catherine\u00a0B.C. Schaumans Stijn Kelchtermans Chan Naseeb S. Mason\u00a0Garrison Tal Yarkoni C.S. Richard\u00a0Chan Prestone Adie Paulius Alaburda Casper Albers Sara Alspaugh Jeff Alstott Andrew\u00a0A. Nelson Eduardo Ari\u00f1o de\u00a0la Rubia Adbi Arzi \u0160t\u011bp\u00e1n Bahn\u00edk Jason Baik Laura Winther\u00a0Balling Sachin Banker David AA\u00a0Baranger Dale\u00a0J. Barr Brenda Barros-Rivera Matt Bauer Enuh Blaise Lisa Boelen Katerina Bohle\u00a0Carbonell Robert\u00a0A. Briers Oliver Burkhard Miguel-Angel Canela Laura Castrillo Timothy Catlett Olivia Chen Michael Clark Brent Cohn Alex Coppock Nat\u00e0lia Cuguer\u00f3-Escofet Paul\u00a0G. Curran Wilson Cyrus-Lai David Dai Giulio Valentino Dalla\u00a0Riva Henrik Danielsson Rosaria de\u00a0F.S.M. Russo Niko de Silva Curdin Derungs Frank Dondelinger Carolina Duarte\u00a0de Souza B. Tyson\u00a0Dube Marina Dubova Ben Mark\u00a0Dunn Peter Adriaan\u00a0Edelsbrunner Sara Finley Nick Fox Timo Gnambs Yuanyuan Gong Erin Grand Brandon Greenawalt Dan Han Paul\u00a0H.P. Hanel Antony\u00a0B. Hong David Hood Justin Hsueh Lilian Huang Kent\u00a0N. Hui Keith\u00a0A. Hultman Azka Javaid Lily Ji\u00a0Jiang Jonathan Jong Jash Kamdar David Kane Gregor Kappler Erikson Kaszubowski Christopher\u00a0M. Kavanagh Madian Khabsa Bennett Kleinberg Jens Kouros Heather Krause Angelos-Miltiadis Krypotos Dejan Lavbi\u010d Rui Ling\u00a0Lee Timothy Leffel Wei Yang\u00a0Lim Silvia Liverani Bianca Loh Dorte L\u00f8nsmann Jia Wei\u00a0Low Alton Lu Kyle MacDonald Christopher\u00a0R. Madan Lasse Hjorth\u00a0Madsen Christina Maimone Alexandra Mangold Adrienne Marshall Helena Ester\u00a0Matskewich Kimia Mavon Katherine\u00a0L. McLain Amelia\u00a0A. McNamara Mhairi McNeill Ulf Mertens David Miller Ben Moore Andrew Moore Eric Nantz Ziauddin Nasrullah Valentina Nejkovic Colleen\u00a0S Nell Andrew Arthur\u00a0Nelson Gustav Nilsonne Rory Nolan Christopher\u00a0E. O\u2019Brien Patrick O\u2019Neill Kieran O\u2019Shea Toto Olita Jahna Otterbacher Diana Palsetia Bianca Pereira Ivan Pozdniakov John Protzko Jean-Nicolas Reyt Travis Riddle Amal (Akmal) Ridhwan Omar\u00a0Ali Ivan Ropovik Joshua\u00a0M. Rosenberg Stephane Rothen Michael Schulte-Mecklenbeck Nirek Sharma Gordon Shotwell Martin Skarzynski William Stedden Victoria Stodden Martin\u00a0A. Stoffel Scott Stoltzman Subashini Subbaiah Rachael Tatman Paul\u00a0H. Thibodeau Sabina Tomkins Ana Valdivia Gerrieke\u00a0B. Druijff-van\u00a0de Woestijne Laura Viana Florence Villes\u00e8che W. Duncan\u00a0Wadsworth Florian Wanders Krista Watts Jason\u00a0D Wells Christopher\u00a0E. Whelpley Andy Won Lawrence Wu Arthur Yip Casey Youngflesh Ju-Chi Yu Arash Zandian Leilei Zhang Chava Zibman and Eric Luis\u00a0Uhlmann. 2021. Same data different conclusions: Radical dispersion in empirical results when independent analysts operationalize and test the same hypothesis. Organizational Behavior and Human Decision Processes 165 (July 2021) 228\u2013249. https:\/\/doi.org\/10.1016\/j.obhdp.2021.02.003","DOI":"10.1016\/j.obhdp.2021.02.003"},{"key":"e_1_3_3_3_51_1","unstructured":"R. Silberzahn E.\u00a0L. Uhlmann D.\u00a0P. Martin P. Anselmi F. Aust E. Awtrey \u0160. Bahn\u00edk F. Bai C. Bannard E. Bonnier R. Carlsson F. Cheung G. Christensen R. Clay M.\u00a0A. Craig A. Dalla\u00a0Rosa L. Dam M.\u00a0H. Evans I. Flores\u00a0Cervantes N. Fong M. Gamez-Djokic A. Glenz S. Gordon-McKeon T.\u00a0J. Heaton K. Hederos M. Heene A.\u00a0J. Hofelich\u00a0Mohr F. H\u00f6gden K. Hui M. Johannesson J. Kalodimos E. Kaszubowski D.\u00a0M. Kennedy"},{"key":"e_1_3_3_3_52_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.2694998"},{"key":"e_1_3_3_3_53_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-020-0912-z"},{"key":"e_1_3_3_3_54_1","doi-asserted-by":"publisher","DOI":"10.1111\/1740-9713.01672"},{"key":"e_1_3_3_3_55_1","doi-asserted-by":"publisher","DOI":"10.1177\/1745691616658637"},{"key":"e_1_3_3_3_56_1","volume-title":"multiverse analysis. (May","author":"Vanpaemel Wolf","year":"2016","unstructured":"Wolf Vanpaemel, Sara Steegen, Francis Tuerlinckx, and Andrew Gelman. 2016. multiverse analysis. (May 2016). https:\/\/osf.io\/zj68b\/ Publisher: OSF."},{"key":"e_1_3_3_3_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2017.2734091"}],"event":{"name":"CHI '23: CHI Conference on Human Factors in Computing Systems","location":"Hamburg Germany","acronym":"CHI '23","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581099","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3544548.3581099","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:57Z","timestamp":1750178877000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3544548.3581099"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,19]]},"references-count":56,"alternative-id":["10.1145\/3544548.3581099","10.1145\/3544548"],"URL":"https:\/\/doi.org\/10.1145\/3544548.3581099","relation":{},"subject":[],"published":{"date-parts":[[2023,4,19]]},"assertion":[{"value":"2023-04-19","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}