{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,23]],"date-time":"2026-04-23T07:57:38Z","timestamp":1776931058083,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":109,"publisher":"ACM","license":[{"start":{"date-parts":[[2027,3,22]],"date-time":"2027-03-22T00:00:00Z","timestamp":1805673600000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF (National Science Foundation)","doi-asserted-by":"publisher","award":["IIS-2402647"],"award-info":[{"award-number":["IIS-2402647"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006785","name":"Google","doi-asserted-by":"publisher","award":["Gift"],"award-info":[{"award-number":["Gift"]}],"id":[{"id":"10.13039\/100006785","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,3,23]]},"DOI":"10.1145\/3742413.3789160","type":"proceedings-article","created":{"date-parts":[[2026,3,3]],"date-time":"2026-03-03T11:32:24Z","timestamp":1772537544000},"page":"683-698","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["RDoFlow: Automatically assessing under-specified statistical analyses in HCI"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7290-068X","authenticated-orcid":false,"given":"Madeleine","family":"Grunde-McLaughlin","sequence":"first","affiliation":[{"name":"University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-3908-1572","authenticated-orcid":false,"given":"Weixuan","family":"Liu","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-0671-2104","authenticated-orcid":false,"given":"Ria","family":"Patil","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-0587-2100","authenticated-orcid":false,"given":"Nino","family":"Migineishvili","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3572-6234","authenticated-orcid":false,"given":"Emily","family":"Reif","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8784-2531","authenticated-orcid":false,"given":"Ranjay","family":"Krishna","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, Washington, USA and Allen Institute for AI, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3255-0109","authenticated-orcid":false,"given":"Daniel S.","family":"Weld","sequence":"additional","affiliation":[{"name":"Allen Institute for AI, Seattle, Washington, USA and University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6175-1655","authenticated-orcid":false,"given":"Jeffrey","family":"Heer","sequence":"additional","affiliation":[{"name":"University of Washington, Seattle, Washington, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2026,3,22]]},"reference":[{"key":"e_1_3_3_3_2_2","unstructured":"[n. d.]. https:\/\/aspredicted.org\/"},{"key":"e_1_3_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"e_1_3_3_3_4_2","doi-asserted-by":"crossref","unstructured":"Marjan Bakker Coosje\u00a0LS Veldkamp Marcel\u00a0ALM van Assen Elise\u00a0AV Crompvoets How\u00a0Hwee Ong Brian\u00a0A Nosek Courtney\u00a0K Soderberg David Mellor and Jelte\u00a0M Wicherts. 2020. Ensuring the quality and specificity of preregistrations. PLoS biology 18 12 (2020) e3000937.","DOI":"10.1371\/journal.pbio.3000937"},{"key":"e_1_3_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.5555\/2906831.2906897"},{"key":"e_1_3_3_3_6_2","unstructured":"Samuel\u00a0J. Bell and Onno\u00a0Pepijn Kampman. 2021. Perspectives on Machine Learning from Psychology\u2019s Reproducibility Crisis. CoRR abs\/2104.08878 (2021). arXiv:https:\/\/arXiv.org\/abs\/2104.08878https:\/\/arxiv.org\/abs\/2104.08878"},{"key":"e_1_3_3_3_7_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.findings-acl.226"},{"key":"e_1_3_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290607.3310432"},{"key":"e_1_3_3_3_9_2","doi-asserted-by":"crossref","unstructured":"Graeme Blair Jasper Cooper Alexander Coppock and Macartan Humphreys. 2019. Declaring and diagnosing research designs. American Political Science Review 113 3 (2019) 838\u2013859.","DOI":"10.1017\/S0003055419000194"},{"key":"e_1_3_3_3_10_2","doi-asserted-by":"crossref","unstructured":"Zana Bu\u00e7inca Maja\u00a0Barbara Malaya and Krzysztof\u00a0Z Gajos. 2021. To trust or to think: cognitive forcing functions can reduce overreliance on AI in AI-assisted decision-making. Proceedings of the ACM on Human-computer Interaction 5 CSCW1 (2021) 1\u201321.","DOI":"10.1145\/3449287"},{"key":"e_1_3_3_3_11_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2015.26"},{"key":"e_1_3_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"e_1_3_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.14236\/ewic\/HCI2007.20"},{"key":"e_1_3_3_3_14_2","doi-asserted-by":"crossref","unstructured":"Colin\u00a0F Camerer Anna Dreber Eskil Forsell Teck-Hua Ho J\u00fcrgen Huber Magnus Johannesson Michael Kirchler Johan Almenberg Adam Altmejd Taizan Chan et\u00a0al. 2016. Evaluating replicability of laboratory experiments in economics. Science 351 6280 (2016) 1433\u20131436.","DOI":"10.1126\/science.aaf0918"},{"key":"e_1_3_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.580"},{"key":"e_1_3_3_3_16_2","doi-asserted-by":"publisher","unstructured":"Andy Cockburn Pierre Dragicevic Lonni Besan\u00e7on and Carl Gutwin. 2020. Threats of a replication crisis in empirical computer science. Commun. ACM 63 8 (July 2020) 70\u201379. 10.1145\/3360311","DOI":"10.1145\/3360311"},{"key":"e_1_3_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173715"},{"key":"e_1_3_3_3_18_2","doi-asserted-by":"publisher","unstructured":"Open\u00a0Science Collaboration. 2015. Estimating the reproducibility of psychological science. Science 349 6251 (2015) aac4716. arXiv:https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.aac471610.1126\/science.aac4716","DOI":"10.1126\/science.aac4716"},{"key":"e_1_3_3_3_19_2","doi-asserted-by":"crossref","unstructured":"Sean\u00a0D Connell and Jonathan\u00a0YS Leung. 2023. Reproducibility crisis and gravitation towards a consensus in ocean acidification research. Nature Climate Change 13 11 (2023) 1266\u20131271.","DOI":"10.1038\/s41558-023-01828-9"},{"key":"e_1_3_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1109\/RO-MAN53752.2022.9900744"},{"key":"e_1_3_3_3_21_2","unstructured":"Antonia Creswell Murray Shanahan and Irina Higgins. 2022. Selection-inference: Exploiting large language models for interpretable logical reasoning. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2205.09712 (2022)."},{"key":"e_1_3_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-26633-6_13"},{"key":"e_1_3_3_3_23_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300295"},{"key":"e_1_3_3_3_24_2","series-title":"Proceedings of Machine Learning Research","first-page":"11733","volume-title":"Proceedings of the 41st International Conference on Machine Learning","volume":"235","author":"Du Yilun","year":"2024","unstructured":"Yilun Du, Shuang Li, Antonio Torralba, Joshua\u00a0B. Tenenbaum, and Igor Mordatch. 2024. Improving Factuality and Reasoning in Language Models through Multiagent Debate. In Proceedings of the 41st International Conference on Machine Learning(Proceedings of Machine Learning Research, Vol.\u00a0235), Ruslan Salakhutdinov, Zico Kolter, Katherine Heller, Adrian Weller, Nuria Oliver, Jonathan Scarlett, and Felix Berkenkamp (Eds.). PMLR, 11733\u201311763. https:\/\/proceedings.mlr.press\/v235\/du24e.html"},{"key":"e_1_3_3_3_25_2","doi-asserted-by":"crossref","unstructured":"Nouha Dziri Sivan Milton Mo Yu Osmar\u00a0R Zaiane and Siva Reddy. 2022. On the Origin of Hallucinations in Conversational Models: Is it the Datasets or the Models?ArXiv abs\/2204.07931 (2022). https:\/\/api.semanticscholar.org\/CorpusID:248227301","DOI":"10.18653\/v1\/2022.naacl-main.387"},{"key":"e_1_3_3_3_26_2","doi-asserted-by":"publisher","DOI":"10.1145\/3170427.3188395"},{"key":"e_1_3_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300447"},{"key":"e_1_3_3_3_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300690"},{"key":"e_1_3_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290607.3312905"},{"key":"e_1_3_3_3_30_2","doi-asserted-by":"crossref","unstructured":"Erin\u00a0D Foster and Ariel Deardorff. 2017. Open science framework (OSF). Journal of the Medical Library Association: JMLA 105 2 (2017) 203.","DOI":"10.5195\/jmla.2017.88"},{"key":"e_1_3_3_3_31_2","unstructured":"Andrew Gelman and Eric Loken. 2013. The garden of forking paths: Why multiple comparisons can be a problem even when there is no \u201cfishing expedition\u201d or \u201cp-hacking\u201d and the research hypothesis was posited ahead of time. Department of Statistics Columbia University 348 1-17 (2013) 3."},{"key":"e_1_3_3_3_32_2","unstructured":"Zelalem Gero Chandan Singh Hao Cheng Tristan Naumann Michel Galley Jianfeng Gao and Hoifung Poon. 2023. Self-Verification Improves Few-Shot Clinical Information Extraction. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2306.00024 (2023)."},{"key":"e_1_3_3_3_33_2","unstructured":"Christian Greiffenhagen and Stuart Reeves. 2013. Is Replication important for HCI?CEUR Workshop Proceedings 976."},{"key":"e_1_3_3_3_34_2","doi-asserted-by":"crossref","unstructured":"Madeleine Grunde-McLaughlin Michelle\u00a0S Lam Ranjay Krishna Daniel\u00a0S Weld and Jeffrey Heer. 2025. Designing LLM chains by adapting techniques from crowdsourcing workflows. ACM Transactions on Computer-Human Interaction 32 3 (2025) 1\u201357.","DOI":"10.1145\/3716134"},{"key":"e_1_3_3_3_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3641891"},{"key":"e_1_3_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11503"},{"key":"e_1_3_3_3_37_2","doi-asserted-by":"crossref","unstructured":"Hatice Gunes Frank Broz Chris\u00a0S Crawford Astrid Rosenthal-von der P\u00fctten Megan Strait and Laurel Riek. 2022. Reproducibility in human-robot interaction: Furthering the science of HRI. Current Robotics Reports 3 4 (2022) 281\u2013292.","DOI":"10.1007\/s43154-022-00094-5"},{"key":"e_1_3_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.14443"},{"key":"e_1_3_3_3_39_2","doi-asserted-by":"crossref","unstructured":"Megan\u00a0L Head Luke Holman Rob Lanfear Andrew\u00a0T Kahn and Michael\u00a0D Jennions. 2015. The extent and consequences of p-hacking in science. PLoS biology 13 3 (2015) e1002106.","DOI":"10.1371\/journal.pbio.1002106"},{"key":"e_1_3_3_3_40_2","doi-asserted-by":"crossref","unstructured":"Robert Heirene Debi LaPlante Eric Louderback Brittany Keen Marjan Bakker Anastasia Serafimovska and Sally Gainsbury. 2024. Preregistration specificity and adherence: A review of preregistered gambling studies and cross-disciplinary comparison. Meta-Psychology 8 (2024).","DOI":"10.15626\/MP.2021.2909"},{"key":"e_1_3_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISMAR-Adjunct54149.2021.00030"},{"key":"e_1_3_3_3_42_2","doi-asserted-by":"crossref","unstructured":"Sabine Hoffmann Felix Sch\u00f6nbrodt Ralf Elsas Rory Wilson Ulrich Strasser and Anne-Laure Boulesteix. 2021. The multiplicity of analysis strategies jeopardizes replicability: lessons learned across disciplines. Royal Society open science 8 4 (2021) 201925.","DOI":"10.1098\/rsos.201925"},{"key":"e_1_3_3_3_43_2","doi-asserted-by":"crossref","unstructured":"Ari Holtzman Peter West Vered Shwartz Yejin Choi and Luke Zettlemoyer. 2021. Surface form competition: Why the highest probability answer isn\u2019t always right. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2104.08315 (2021).","DOI":"10.18653\/v1\/2021.emnlp-main.564"},{"key":"e_1_3_3_3_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557004"},{"key":"e_1_3_3_3_45_2","doi-asserted-by":"crossref","unstructured":"Robert Hudson. 2021. Should we strive to make science bias-free? A philosophical assessment of the reproducibility crisis. Journal for General Philosophy of Science 52 3 (2021) 389\u2013405.","DOI":"10.1007\/s10838-020-09548-w"},{"key":"e_1_3_3_3_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3514094.3534196"},{"key":"e_1_3_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2025.naacl-demo.44"},{"key":"e_1_3_3_3_48_2","doi-asserted-by":"crossref","unstructured":"John\u00a0PA Ioannidis. 2005. Why most published research findings are false. PLoS medicine 2 8 (2005) e124.","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_3_3_49_2","doi-asserted-by":"publisher","unstructured":"Eunice Jun Melissa Birchfield Nicole De\u00a0Moura Jeffrey Heer and Ren\u00e9 Just. 2022. Hypothesis Formalization: Empirical Findings Software Limitations and Design Implications. ACM Trans. Comput.-Hum. Interact. 29 1 Article 6 (Jan. 2022) 28\u00a0pages. 10.1145\/3476980","DOI":"10.1145\/3476980"},{"key":"e_1_3_3_3_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/3332165.3347940"},{"key":"e_1_3_3_3_51_2","doi-asserted-by":"crossref","unstructured":"Robert\u00a0M Kaplan and Veronica\u00a0L Irvin. 2015. Likelihood of null effects of large NHLBI clinical trials has increased over time. PloS one 10 8 (2015) e0132382.","DOI":"10.1371\/journal.pone.0132382"},{"key":"e_1_3_3_3_52_2","doi-asserted-by":"crossref","unstructured":"Sayash Kapoor and Arvind Narayanan. 2023. Leakage and the reproducibility crisis in machine-learning-based science. Patterns 4 9 (2023).","DOI":"10.1016\/j.patter.2023.100804"},{"key":"e_1_3_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208557"},{"key":"e_1_3_3_3_54_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858465"},{"key":"e_1_3_3_3_55_2","doi-asserted-by":"crossref","unstructured":"Norbert\u00a0L Kerr. 1998. HARKing: Hypothesizing after the results are known. Personality and social psychology review 2 3 (1998) 196\u2013217.","DOI":"10.1207\/s15327957pspr0203_4"},{"key":"e_1_3_3_3_56_2","unstructured":"Jaeho Kim Yunseok Lee and Seulki Lee. 2025. Position: The AI Conference Peer Review Crisis Demands Author Feedback and Reviewer Rewards. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2505.04966 (2025)."},{"key":"e_1_3_3_3_57_2","doi-asserted-by":"publisher","DOI":"10.1145\/2993901.2993909"},{"key":"e_1_3_3_3_58_2","doi-asserted-by":"publisher","DOI":"10.1109\/BELIV.2018.8634392"},{"key":"e_1_3_3_3_59_2","doi-asserted-by":"publisher","unstructured":"Gavin Leech Juan\u00a0J. Vazquez Misha Yagudin Niclas Kupper and Laurence Aitchison. 2024. Questionable practices in machine learning. CoRR abs\/2407.12220 (2024). 10.48550\/arXiv.2407.12220","DOI":"10.48550\/arXiv.2407.12220"},{"key":"e_1_3_3_3_60_2","doi-asserted-by":"crossref","unstructured":"Benedikt Leichtmann Verena Nitsch and Martina Mara. 2022. Crisis ahead? Why human-robot interaction user studies may have replicability problems and directions for improvement. Frontiers in Robotics and AI 9 (2022) 838116.","DOI":"10.3389\/frobt.2022.838116"},{"key":"e_1_3_3_3_61_2","series-title":"(NIPS \u201923)","volume-title":"Proceedings of the 37th International Conference on Neural Information Processing Systems","author":"Li Guohao","year":"2024","unstructured":"Guohao Li, Hasan\u00a0Abed Al\u00a0Kader\u00a0Hammoud, Hani Itani, Dmitrii Khizbullin, and Bernard Ghanem. 2024. CAMEL: communicative agents for \"mind\" exploration of large language model society. In Proceedings of the 37th International Conference on Neural Information Processing Systems (New Orleans, LA, USA) (NIPS \u201923). Curran Associates Inc., Red Hook, NY, USA, Article 2264, 18\u00a0pages."},{"key":"e_1_3_3_3_62_2","unstructured":"Long Li Weiwen Xu Jiayan Guo Ruochen Zhao Xingxuan Li Yuqian Yuan Boqiang Zhang Yuming Jiang Yifei Xin Ronghao Dang et\u00a0al. 2024. Chain of ideas: Revolutionizing research via novel idea development with llm agents. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2410.13185 (2024)."},{"key":"e_1_3_3_3_63_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.emnlp-main.992"},{"key":"e_1_3_3_3_64_2","doi-asserted-by":"publisher","unstructured":"Jiali Liu Nadia Boukhelifa and James\u00a0R. Eagan. 2020. Understanding the Role of Alternatives in Data Analysis Practices. IEEE Transactions on Visualization and Computer Graphics 26 1 (2020) 66\u201376. 10.1109\/TVCG.2019.2934593","DOI":"10.1109\/TVCG.2019.2934593"},{"key":"e_1_3_3_3_65_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376533"},{"key":"e_1_3_3_3_66_2","doi-asserted-by":"publisher","unstructured":"Yang Liu Alex Kale Tim Althoff and Jeffrey Heer. 2021. Boba: Authoring and Visualizing Multiverse Analyses. IEEE Transactions on Visualization and Computer Graphics 27 2 (2021) 1753\u20131763. 10.1109\/TVCG.2020.3028985","DOI":"10.1109\/TVCG.2020.3028985"},{"key":"e_1_3_3_3_67_2","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240840"},{"key":"e_1_3_3_3_68_2","doi-asserted-by":"crossref","unstructured":"Joshua Maynez Shashi Narayan Bernd Bohnet and Ryan McDonald. 2020. On faithfulness and factuality in abstractive summarization. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2005.00661 (2020).","DOI":"10.18653\/v1\/2020.acl-main.173"},{"key":"e_1_3_3_3_69_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-67687-6_10"},{"key":"e_1_3_3_3_70_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581225"},{"key":"e_1_3_3_3_71_2","doi-asserted-by":"crossref","unstructured":"Tsuyoshi Miyakawa. 2020. No raw data no science: another possible source of the reproducibility crisis. Molecular brain 13 1 (2020) 24.","DOI":"10.1186\/s13041-020-0552-2"},{"key":"e_1_3_3_3_72_2","doi-asserted-by":"crossref","unstructured":"Kevin\u00a0J Mullinix Thomas\u00a0J Leeper James\u00a0N Druckman and Jeremy Freese. 2015. The generalizability of survey experiments. Journal of Experimental Political Science 2 2 (2015) 109\u2013138.","DOI":"10.1017\/XPS.2015.19"},{"key":"e_1_3_3_3_73_2","unstructured":"Varun Nair Elliot Schumacher Geoffrey Tso and Anitha Kannan. 2023. DERA: enhancing large language model completions with dialog-enabled resolving agents. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.17071 (2023)."},{"key":"e_1_3_3_3_74_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.clinicalnlp-1.12"},{"key":"e_1_3_3_3_75_2","unstructured":"Feng Nie Meixi Chen Zhirui Zhang and Xu Cheng. 2022. Improving few-shot performance of language models via nearest neighbor calibration. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2212.02216 (2022)."},{"key":"e_1_3_3_3_76_2","volume-title":"GPT-4o System Card","year":"2024","unstructured":"OpenAI. 2024. GPT-4o System Card. Technical Report. OpenAI. https:\/\/cdn.openai.com\/gpt-4o-system-card.pdf"},{"key":"e_1_3_3_3_77_2","volume-title":"GPT-5 System Card","year":"2025","unstructured":"OpenAI. 2025. GPT-5 System Card. Technical Report. OpenAI. https:\/\/cdn.openai.com\/gpt-5-system-card.pdf"},{"key":"e_1_3_3_3_78_2","doi-asserted-by":"publisher","DOI":"10.1145\/3706598.3714022"},{"key":"e_1_3_3_3_79_2","series-title":"(SOUPS \u201923)","volume-title":"Proceedings of the Nineteenth USENIX Conference on Usable Privacy and Security","author":"Ortloff Anna-Marie","year":"2023","unstructured":"Anna-Marie Ortloff, Christian Tiefenau, and Matthew Smith. 2023. SoK: i have the (developer) power! sample size estimation for Fisher\u2019s exact, Chi-Squared, McNemar\u2019s, Wilcoxon rank-sum, Wilcoxon signed-rank and t-tests in developer-centered usable security. In Proceedings of the Nineteenth USENIX Conference on Usable Privacy and Security (Anaheim, CA, USA) (SOUPS \u201923). USENIX Association, USA, Article 19, 19\u00a0pages."},{"key":"e_1_3_3_3_80_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858283"},{"key":"e_1_3_3_3_81_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517707"},{"key":"e_1_3_3_3_82_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290607.3312862"},{"key":"e_1_3_3_3_83_2","doi-asserted-by":"crossref","unstructured":"Felix Putze Susanne Putze Merle Sagehorn Christopher Micek and Erin\u00a0T Solovey. 2022. Understanding hci practices and challenges of experiment reporting with brain signals: Towards reproducibility and reuse. ACM Transactions on Computer-Human Interaction (TOCHI) 29 4 (2022) 1\u201343.","DOI":"10.1145\/3490554"},{"key":"e_1_3_3_3_84_2","unstructured":"Ghulam\u00a0Jilani Quadri and Paul Rosen. 2019. You can\u2019t publish replication studies (and how to anyways). arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/1908.08893 (2019)."},{"key":"e_1_3_3_3_85_2","doi-asserted-by":"crossref","unstructured":"Robert Rosenthal. 1979. The file drawer problem and tolerance for null results.Psychological bulletin 86 3 (1979) 638.","DOI":"10.1037\/0033-2909.86.3.638"},{"key":"e_1_3_3_3_86_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580848"},{"key":"e_1_3_3_3_87_2","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642375"},{"key":"e_1_3_3_3_88_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580726"},{"key":"e_1_3_3_3_89_2","unstructured":"Martin Schweinsberg Michael Feldman Nicola Staub Olmo\u00a0R van\u00a0den Akker Robbie\u00a0CM van Aert Marcel\u00a0ALM Van\u00a0Assen Yang Liu Tim Althoff Jeffrey Heer Alex Kale et\u00a0al. 2021. Same data different conclusions: Radical dispersion in empirical results when independent analysts operationalize and test the same hypothesis. Organizational Behavior and Human Decision Processes 165 (2021) 228\u2013249."},{"key":"e_1_3_3_3_90_2","doi-asserted-by":"crossref","unstructured":"Joseph\u00a0P Simmons Leif\u00a0D Nelson and Uri Simonsohn. 2011. False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant. Psychological science 22 11 (2011) 1359\u20131366.","DOI":"10.1177\/0956797611417632"},{"key":"e_1_3_3_3_91_2","doi-asserted-by":"crossref","unstructured":"Susan\u00a0Leigh Star and James\u00a0R Griesemer. 1989. Institutional ecology translations\u2019 and boundary objects: Amateurs and professionals in Berkeley\u2019s Museum of Vertebrate Zoology 1907-39. Social studies of science 19 3 (1989) 387\u2013420.","DOI":"10.1177\/030631289019003001"},{"key":"e_1_3_3_3_92_2","doi-asserted-by":"crossref","unstructured":"Theodore\u00a0D Sterling. 1959. Publication decisions and their possible effects on inferences drawn from tests of significance\u2014or vice versa. Journal of the American statistical association 54 285 (1959) 30\u201334.","DOI":"10.1080\/01621459.1959.10501497"},{"key":"e_1_3_3_3_93_2","doi-asserted-by":"publisher","DOI":"10.1109\/BELIV.2018.8634261"},{"key":"e_1_3_3_3_94_2","doi-asserted-by":"publisher","DOI":"10.1145\/3640543.3645159"},{"key":"e_1_3_3_3_95_2","doi-asserted-by":"crossref","unstructured":"Olmo\u00a0R. van\u00a0den Akker Marjan Bakker Marcel A L\u00a0M van Assen Charlotte\u00a0R. Pennington Leone Verweij Mahmoud\u00a0Medhat Elsherif Aline Claesen Stefan D.\u00a0M. Gaillard Siu\u00a0Kit Yeung Jan-Luca Frankenberger Kai Krautter Jamie\u00a0P. Cockcroft Katharina\u00a0S. Kreuer Thomas\u00a0Rhys Evans Fr\u00e9d\u00e9rique\u00a0M. Heppel Sarah\u00a0F. Schoch Max Korbmacher Yuki Yamada Nihan Albayrak\u2010Aydemir Shilaan Alzahawi Alexandra Sarafoglou Maksim\u00a0M. Sitnikov Filip D\u011bcht\u011brenko Sophia Wingen Sandra Grinschgl Helena Hartmann Suzanne L\u00a0K Stewart Catia M.\u00a0F. de Oliveira Sarah Ashcroft-Jones Bradley\u00a0J. Baker and Jelte\u00a0M. Wicherts. 2024. The potential of preregistration in psychology: Assessing preregistration producibility and preregistration-study consistency.Psychological Methods (2024).","DOI":"10.1037\/met0000687"},{"key":"e_1_3_3_3_96_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W19-8643"},{"key":"e_1_3_3_3_97_2","doi-asserted-by":"crossref","unstructured":"Helena Vasconcelos Matthew J\u00f6rke Madeleine Grunde-McLaughlin Tobias Gerstenberg Michael\u00a0S Bernstein and Ranjay Krishna. 2023. Explanations can reduce overreliance on ai systems during decision-making. Proceedings of the ACM on Human-Computer Interaction 7 CSCW1 (2023) 1\u201338.","DOI":"10.1145\/3579605"},{"key":"e_1_3_3_3_98_2","doi-asserted-by":"publisher","DOI":"10.1145\/3410404.3414229"},{"key":"e_1_3_3_3_99_2","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702347"},{"key":"e_1_3_3_3_100_2","doi-asserted-by":"crossref","unstructured":"Xiaoyi Wang Alexander Eiselmayer Wendy\u00a0E Mackay Kasper Hornbaek and Chat Wacharamanotham. 2020. Argus: Interactive a priori power analysis. IEEE Transactions on Visualization and Computer Graphics 27 2 (2020) 432\u2013442.","DOI":"10.1109\/TVCG.2020.3028894"},{"key":"e_1_3_3_3_101_2","doi-asserted-by":"crossref","unstructured":"Jelte\u00a0M Wicherts Coosje\u00a0LS Veldkamp Hilde\u00a0EM Augusteijn Marjan Bakker Robbie\u00a0CM Van\u00a0Aert and Marcel\u00a0ALM Van\u00a0Assen. 2016. Degrees of freedom in planning running analyzing and reporting psychological studies: A checklist to avoid p-hacking. Frontiers in psychology 7 (2016) 1832.","DOI":"10.3389\/fpsyg.2016.01832"},{"key":"e_1_3_3_3_102_2","doi-asserted-by":"publisher","DOI":"10.1145\/2559206.2559233"},{"key":"e_1_3_3_3_103_2","doi-asserted-by":"publisher","DOI":"10.1145\/2468356.2479636"},{"key":"e_1_3_3_3_104_2","volume-title":"First Conference on Language Modeling","author":"Wu Qingyun","year":"2024","unstructured":"Qingyun Wu, Gagan Bansal, Jieyu Zhang, Yiran Wu, Beibin Li, Erkang Zhu, Li Jiang, Xiaoyun Zhang, Shaokun Zhang, Jiale Liu, et\u00a0al. 2024. Autogen: Enabling next-gen LLM applications via multi-agent conversations. In First Conference on Language Modeling."},{"key":"e_1_3_3_3_105_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517582"},{"key":"e_1_3_3_3_106_2","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.emnlp-main.296"},{"key":"e_1_3_3_3_107_2","unstructured":"Shunyu Yao Dian Yu Jeffrey Zhao Izhak Shafran Tom Griffiths Yuan Cao and Karthik Narasimhan. 2024. Tree of thoughts: Deliberate problem solving with large language models. Advances in Neural Information Processing Systems 36 (2024)."},{"key":"e_1_3_3_3_108_2","doi-asserted-by":"publisher","unstructured":"Ed Yong. 2012. Replication studies: Bad copy. Nature 485 (2012) 298\u2013300. 10.1038\/485298a","DOI":"10.1038\/485298a"},{"key":"e_1_3_3_3_109_2","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3581388"},{"key":"e_1_3_3_3_110_2","doi-asserted-by":"crossref","unstructured":"Ge Zhang Mike\u00a0A Merrill Yang Liu Jeffrey Heer and Tim Althoff. 2022. Coral: Code representation learning with weakly-supervised transformers for analyzing data analysis. EPJ Data Science 11 1 (2022) 14.","DOI":"10.1140\/epjds\/s13688-022-00327-9"}],"event":{"name":"IUI '26: 31st International Conference on Intelligent User Interfaces","location":"Paphos Cyprus","acronym":"IUI '26","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction","SIGAI ACM Special Interest Group on Artificial Intelligence"]},"container-title":["Proceedings of the 31st International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/abs\/10.1145\/3742413.3789160","content-type":"text\/html","content-version":"vor","intended-application":"syndication"}],"deposited":{"date-parts":[[2026,3,14]],"date-time":"2026-03-14T12:55:07Z","timestamp":1773492907000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3742413.3789160"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,3,22]]},"references-count":109,"alternative-id":["10.1145\/3742413.3789160","10.1145\/3742413"],"URL":"https:\/\/doi.org\/10.1145\/3742413.3789160","relation":{},"subject":[],"published":{"date-parts":[[2026,3,22]]},"assertion":[{"value":"2026-03-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}