{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,8]],"date-time":"2026-10-08T11:37:07Z","timestamp":1791459427750,"version":"4.3.4"},"publisher-location":"New York, NY, USA","reference-count":60,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T00:00:00Z","timestamp":1726012800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"NSF","award":["CCF-1763788 and CCF-1956374"],"award-info":[{"award-number":["CCF-1763788 and CCF-1956374"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,9,11]]},"DOI":"10.1145\/3650212.3680369","type":"proceedings-article","created":{"date-parts":[[2024,9,11]],"date-time":"2024-09-11T11:44:25Z","timestamp":1726055065000},"page":"1402-1414","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":14,"title":["Neurosymbolic Repair of Test Flakiness"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0008-1409-4163","authenticated-orcid":false,"given":"Yang","family":"Chen","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0668-8526","authenticated-orcid":false,"given":"Reyhaneh","family":"Jabbarvand","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,9,11]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2023. Apache Maven Surefire. https:\/\/github.com\/apache\/maven-surefire"},{"key":"e_1_3_2_1_2_1","unstructured":"2023. The GitHub Repository of FlakyDoctor. https:\/\/github.com\/Intelligent-CAT-Lab\/FlakyDoctor"},{"key":"e_1_3_2_1_4_1","unstructured":"2023. Nondex Test Flakiness Detection Tool. https:\/\/github.com\/TestingResearchIllinois\/NonDex."},{"key":"e_1_3_2_1_5_1","unstructured":"2024. Defects4J Bug Dataset. https:\/\/github.com\/jkoppel\/QuixBugs\/tree\/master"},{"key":"e_1_3_2_1_6_1","unstructured":"2024. International Dataset of Flaky tests. https:\/\/github.com\/TestingResearchIllinois\/idoft"},{"key":"e_1_3_2_1_7_1","unstructured":"2024. QuixBugs Dataset. https:\/\/github.com\/jkoppel\/QuixBugs\/tree\/master"},{"key":"e_1_3_2_1_8_1","unstructured":"2024. Repository Hadoop. https:\/\/github.com\/apache\/hadoop"},{"key":"e_1_3_2_1_9_1","unstructured":"2024. Repository shardingsphere-elasticjob. https:\/\/github.com\/apache\/shardingsphere-elasticjob"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00140"},{"key":"e_1_3_2_1_11_1","volume-title":"2018 IEEE\/ACM 40th International Conference on Software Engineering. 433\u2013444","author":"Bell Jonathan","year":"2018","unstructured":"Jonathan Bell, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. 2018. DeFlaker: Automatically detecting flaky tests. In 2018 IEEE\/ACM 40th International Conference on Software Engineering. 433\u2013444."},{"key":"e_1_3_2_1_12_1","volume-title":"Language models are few-shot learners. Advances in neural information processing systems, 33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, and Amanda Askell. 2020. Language models are few-shot learners. Advances in neural information processing systems, 33 (2020), 1877\u20131901."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2017.8115695"},{"key":"e_1_3_2_1_14_1","volume-title":"Codet: Code generation with generated tests. arXiv preprint arXiv:2207.10397.","author":"Chen Bei","year":"2022","unstructured":"Bei Chen, Fengji Zhang, Anh Nguyen, Daoguang Zan, Zeqi Lin, Jian-Guang Lou, and Weizhu Chen. 2022. Codet: Code generation with generated tests. arXiv preprint arXiv:2207.10397."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3639478.3641227"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","unstructured":"Yang Chen. 2024. FlakyDoctor Artifact. https:\/\/doi.org\/10.5281\/zenodo.12670050 10.5281\/zenodo.12670050","DOI":"10.5281\/zenodo.12670050"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3643656.3643900"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597926.3598119"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397366"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468615"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338945"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"e_1_3_2_1_23_1","unstructured":"Nan Jiang Kevin Liu Thibaud Lutellier and Lin Tan. 2023. Impact of Code Language Models on Automated Program Repair. arxiv:2302.05020. arxiv:2302.05020"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/QRS-C.2018.00031"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377813.3381370"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330570"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3381749"},{"key":"e_1_3_2_1_28_1","volume-title":"iDFlakies: A framework for detecting and partially classifying flaky tests. In 2019 12th ieee conference on software testing, validation and verification. 312\u2013322","author":"Lam Wing","unstructured":"Wing Lam, Reed Oei, August Shi, Darko Marinov, and Tao Xie. 2019. iDFlakies: A framework for detecting and partially classifying flaky tests. In 2019 12th ieee conference on software testing, validation and verification. 312\u2013322."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE5003.2020.00045"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428270"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3182536"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510173"},{"key":"e_1_3_2_1_33_1","volume-title":"Shizhuo Dylan Zhang, and Reyhaneh Jabbarvand","author":"Liu Changshu","year":"2024","unstructured":"Changshu Liu, Shizhuo Dylan Zhang, and Reyhaneh Jabbarvand. 2024. CodeMind: A Framework to Challenge Large Language Models for Code Reasoning. arXiv preprint arXiv:2402.09664."},{"key":"e_1_3_2_1_34_1","volume-title":"Yuyao Wang, and Lingming Zhang.","author":"Liu Jiawei","year":"2023","unstructured":"Jiawei Liu, Chunqiu Steven Xia, Yuyao Wang, and Lingming Zhang. 2023. Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation. arXiv preprint arXiv:2305.01210."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1162\/tacl_a_00638"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.24215\/16666038.18.e16"},{"key":"e_1_3_2_1_37_1","unstructured":"John Micco. 2017. The state of continuous integration testing@ google."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Shuyin Ouyang Jie M Zhang Mark Harman and Meng Wang. 2023. LLM is Like a Box of Chocolates: the Non-determinism of ChatGPT in Code Generation. arXiv preprint arXiv:2308.02828.","DOI":"10.1145\/3697010"},{"key":"e_1_3_2_1_39_1","volume-title":"Rahul Krishna, Divya Sankar, Lambert Pouguem Wassi, Michele Merler, Boris Sobolev, Raju Pavuluri, Saurabh Sinha, and Reyhaneh Jabbarvand.","author":"Pan Rangeet","year":"2023","unstructured":"Rangeet Pan, Ali Reza Ibrahimzada, Rahul Krishna, Divya Sankar, Lambert Pouguem Wassi, Michele Merler, Boris Sobolev, Raju Pavuluri, Saurabh Sinha, and Reyhaneh Jabbarvand. 2023. Understanding the Effectiveness of Large Language Models in Code Translation. arXiv preprint arXiv:2308.03109."},{"key":"e_1_3_2_1_40_1","unstructured":"Yu Pei Jeongju Sohn Sarra Habchi and Mike Papadakis. 2023. TRaf: Time-based Repair for Asynchronous Wait Flaky Tests in Web Testing. arXiv preprint arXiv:2305.08592."},{"key":"e_1_3_2_1_41_1","volume-title":"2015 30th IEEE\/ACM International Conference on Automated Software Engineering. 149\u2013154","author":"Person Suzette","year":"2015","unstructured":"Suzette Person and Sebastian Elbaum. 2015. Test analysis: Searching for faults in tests (N). In 2015 30th IEEE\/ACM International Conference on Automated Software Engineering. 149\u2013154."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379597.3387482"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSM.1999.792604"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338925"},{"key":"e_1_3_2_1_45_1","volume-title":"International Conference on Machine Learning. 31693\u201331715","author":"Shrivastava Disha","year":"2023","unstructured":"Disha Shrivastava, Hugo Larochelle, and Daniel Tarlow. 2023. Repository-level prompt generation for large language models of code. In International Conference on Machine Learning. 31693\u201331715."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3082424"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510454.3516846"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"crossref","unstructured":"Xingyao Wang Hao Peng Reyhaneh Jabbarvand and Heng Ji. 2023. LeTI: Learning to Generate from Textual Interactions. arXiv preprint arXiv:2305.10314.","DOI":"10.18653\/v1\/2024.findings-naacl.16"},{"key":"e_1_3_2_1_49_1","unstructured":"Yufei Wang Wanjun Zhong Liangyou Li Fei Mi Xingshan Zeng Wenyong Huang Lifeng Shang Xin Jiang and Qun Liu. 2023. Aligning large language models with human: A survey. arXiv preprint arXiv:2307.12966."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510170"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-72016-2_15"},{"key":"e_1_3_2_1_52_1","first-page":"24824","article-title":"Chain-of-thought prompting elicits reasoning in large language models","volume":"35","author":"Wei Jason","year":"2022","unstructured":"Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Fei Xia, Ed Chi, Quoc V Le, and Denny Zhou. 2022. Chain-of-thought prompting elicits reasoning in large language models. Advances in Neural Information Processing Systems, 35 (2022), 24824\u201324837.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_53_1","volume-title":"Magicoder: Source Code Is All You Need. arxiv:2312.02120.","author":"Wei Yuxiang","year":"2023","unstructured":"Yuxiang Wei, Zhe Wang, Jiawei Liu, Yifeng Ding, and Lingming Zhang. 2023. Magicoder: Source Code Is All You Need. arxiv:2312.02120."},{"key":"e_1_3_2_1_54_1","unstructured":"Jules White Quchen Fu Sam Hays Michael Sandborn Carlos Olea Henry Gilbert Ashraf Elnashar Jesse Spencer-Smith and Douglas C Schmidt. 2023. A prompt pattern catalog to enhance prompt engineering with chatgpt. arXiv preprint arXiv:2302.11382."},{"key":"e_1_3_2_1_55_1","unstructured":"Chunqiu Steven Xia and Lingming Zhang. 2023. Conversational automated program repair. arXiv preprint arXiv:2301.13246."},{"key":"e_1_3_2_1_56_1","unstructured":"Chunqiu Steven Xia and Lingming Zhang. 2023. Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT. arXiv preprint arXiv:2304.00385."},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623337"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468744.3468756"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00018"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/2610384.2610404"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME46990.2020.00083"}],"event":{"name":"ISSTA '24: 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","location":"Vienna Austria","acronym":"ISSTA '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","AITO"]},"container-title":["Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680369","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3650212.3680369","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:50:08Z","timestamp":1750287008000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3650212.3680369"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9,11]]},"references-count":60,"alternative-id":["10.1145\/3650212.3680369","10.1145\/3650212"],"URL":"https:\/\/doi.org\/10.1145\/3650212.3680369","relation":{},"subject":[],"published":{"date-parts":[[2024,9,11]]},"assertion":[{"value":"2024-09-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}