{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T14:45:38Z","timestamp":1781621138986,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,6,27]],"date-time":"2020-06-27T00:00:00Z","timestamp":1593216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,6,27]]},"DOI":"10.1145\/3377811.3381749","type":"proceedings-article","created":{"date-parts":[[2020,10,1]],"date-time":"2020-10-01T18:25:34Z","timestamp":1601576734000},"page":"1471-1482","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":99,"title":["A study on the lifecycle of flaky tests"],"prefix":"10.1145","author":[{"given":"Wing","family":"Lam","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"K\u0131van\u00e7","family":"Mu\u015flu","sequence":"additional","affiliation":[{"name":"Microsoft"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hitesh","family":"Sajnani","sequence":"additional","affiliation":[{"name":"Microsoft"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Suresh","family":"Thummalapenta","sequence":"additional","affiliation":[{"name":"Microsoft"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2020,10]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2020. Azure Data Explorer. https:\/\/docs.microsoft.com\/en-us\/azure\/data-explorer."},{"key":"e_1_3_2_1_2_1","unstructured":"2020. Bazel. https:\/\/bazel.build."},{"key":"e_1_3_2_1_3_1","unstructured":"2020. Buck. https:\/\/buckbuild.com."},{"key":"e_1_3_2_1_4_1","unstructured":"2020. Data used by \"A Study on the Lifecycle of Flaky Tests\". https:\/\/github.com\/winglam\/flaky-test-lifecycle-data."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"crossref","unstructured":"Jonathan Bell and Gail Kaiser. 2014. Unit test virtualization with VMVM. In ICSE. Hyderabad India 550--561.","DOI":"10.1145\/2568225.2568248"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180164"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Moritz Eck Fabio Palomba Marco Castelluccio and Alberto Bacchelli. 2019. Understanding flaky tests: The developer's perspective. In ESEC\/FSE. Tallinn Estonia 830--840.","DOI":"10.1145\/3338906.3338945"},{"key":"e_1_3_2_1_8_1","volume-title":"PLDI.","author":"Elmas Tayfun","unstructured":"Tayfun Elmas, Jacob Burnim, George Necula, and Koushik Sen. 2013. CONCUR-RIT: A domain specific language for reproducing concurrency bugs. In PLDI. Seattle, WA, USA, 153--164."},{"key":"e_1_3_2_1_9_1","volume-title":"ICSE.","author":"Esfahani Hamed","unstructured":"Hamed Esfahani, Jonas Fietz, Qi Ke, Alexei Kolomiets, Erica Lan, Erik Mavrinac, Wolfram Schulte, Newton Sanches, and Srikanth Kandula. 2016. CloudBuild: Microsoft's distributed and caching build service. In ICSE. Austin, TX, USA, 11--20."},{"key":"e_1_3_2_1_10_1","unstructured":"Martin Fowler. 2020. Eradicating non-determinism in tests. https:\/\/martinfowler.com\/articles\/nonDeterminism.html."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"Alessio Gambi Jonathan Bell and Andreas Zeller. 2018. Practical test dependency detection. In ICST. V\u00e4ster\u00e5s Sweden 1--11.","DOI":"10.1109\/ICST.2018.00011"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Zebao Gao Yalan Liang Myra B. Cohen Atif M. Memon and Zhen Wang. 2015. Making system user interactive tests repeatable: When and what should we control?. In ICSE. Florence Italy 55--65.","DOI":"10.1109\/ICSE.2015.28"},{"key":"e_1_3_2_1_13_1","volume-title":"ISSTA.","author":"Gyori Alex","unstructured":"Alex Gyori, August Shi, Farah Hariri, and Darko Marinov. 2015. Reliable testing: Detecting state-polluting tests to prevent test dependency. In ISSTA. Baltimore, MD, USA, 223--233."},{"key":"e_1_3_2_1_14_1","volume-title":"SCAM.","author":"Harman Mark","unstructured":"Mark Harman and Peter O'Hearn. 2018. From start-ups to scale-ups: Opportunities and open problems for static and dynamic program analysis. In SCAM. Madrid, Spain, 1--23."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Kim Herzig Michaela Greiler Jacek Czerwonka and Brendan Murphy. 2015. The art of testing less without sacrificing quality. In ICSE. Florence Italy 483--493.","DOI":"10.1109\/ICSE.2015.66"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Vilas Jagannath Milos Gligoric Dongyun Jin Qingzhou Luo Grigore Ro\u015fu and Darko Marinov. 2011. Improved multithreaded unit testing. In ESEC\/FSE. Szeged Hungary 223--233.","DOI":"10.1145\/2025113.2025145"},{"key":"e_1_3_2_1_17_1","volume-title":"ESEC\/FSE.","author":"Labuschagne Adriaan","unstructured":"Adriaan Labuschagne, Laura Inozemtseva, and Reid Holmes. 2017. Measuring the cost of regression testing in practice: A study of Java projects using continuous integration. In ESEC\/FSE. Paderborn, Germany, 821--830."},{"key":"e_1_3_2_1_18_1","volume-title":"ISSTA.","author":"Lam Wing","unstructured":"Wing Lam, Patrice Godefroid, Suman Nath, Anirudh Santhiar, and Suresh Thummalapenta. 2019. Root causing flaky tests in a large-scale industrial setting. In ISSTA. Beijing, China, 101--111."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Wing Lam Reed Oei August Shi Darko Marinov and Tao Xie. 2019. iDFlakies: A framework for detecting and partially classifying flaky tests. In ICST. Xi'an China 312--322.","DOI":"10.1109\/ICST.2019.00038"},{"key":"e_1_3_2_1_20_1","unstructured":"Qingzhou Luo Farah Hariri Lamyaa Eloussi and Darko Marinov. 2014. An empirical analysis of flaky tests. In FSE. Hong Kong 643--653."},{"key":"e_1_3_2_1_21_1","volume-title":"ISSTA.","author":"Luo Qingzhou","unstructured":"Qingzhou Luo and Grigore Ro\u015fu. 2013. EnforceMOP: A runtime property enforcement system for multithreaded programs. In ISSTA. Lugano, Switzerland, 156--166."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2647508.2647523"},{"key":"e_1_3_2_1_23_1","volume-title":"ICST.","author":"Micco John","unstructured":"John Micco. 2017. The state of continuous integration testing at Google. In ICST. Tokyo, Japan. https:\/\/bit.ly\/2OohAip"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Kivan\u00e7 Mu\u015flu Bilge Soran and Jochen Wuttke. 2011. Finding bugs by isolating unit tests. In ESEC\/FSE. Szeged Hungary 496--499.","DOI":"10.1145\/2025113.2025202"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Fabio Palomba and Andy Zaidman. 2017. Does refactoring of test smells induce fixing flaky tests?. In ICSME. Shanghai China 1--12.","DOI":"10.1109\/ICSME.2017.12"},{"key":"e_1_3_2_1_26_1","volume-title":"Rigby","author":"Rahman Md Tajmilur","year":"2018","unstructured":"Md Tajmilur Rahman and Peter C. Rigby. 2018. The impact of failing, flaky, and high failure tests on the number of crash reports associated with Firefox builds. In ESEC\/FSE. Lake Buena Vista, FL, USA, 857--862."},{"key":"e_1_3_2_1_27_1","volume-title":"ICST.","author":"Shi August","unstructured":"August Shi, Alex Gyori, Owolabi Legunsen, and Darko Marinov. 2016. Detecting assumptions on deterministic implementations of non-deterministic specifications. In ICST. Chicago, IL, USA, 80--90."},{"key":"e_1_3_2_1_28_1","unstructured":"August Shi Wing Lam Reed Oei Tao Xie and Darko Marinov. 2019. iFixFlakies: A framework for automatically fixing order-dependent flaky tests. In ESEC\/FSE. Tallinn Estonia 545--555."},{"key":"e_1_3_2_1_29_1","volume-title":"ICSME, NIER Track.","author":"Thorve Swapna","unstructured":"Swapna Thorve, Chandani Sreshtha, and Na Meng. 2018. An empirical study of flaky tests in Android apps. In ICSME, NIER Track. Madrid, Spain, 534--538."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3149485.3149494"},{"key":"e_1_3_2_1_31_1","volume-title":"ISSTA.","author":"Zhang Sai","unstructured":"Sai Zhang, Darioush Jalali, Jochen Wuttke, K\u0131van\u00e7 Mu\u015flu, Wing Lam, Michael D. Ernst, and David Notkin. 2014. Empirically revisiting the test independence assumption. In ISSTA. San Jose, CA, USA, 385--396."}],"event":{"name":"ICSE '20: 42nd International Conference on Software Engineering","location":"Seoul South Korea","acronym":"ICSE '20","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","KIISE Korean Institute of Information Scientists and Engineers","IEEE CS"]},"container-title":["Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3381749","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3377811.3381749","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:41:40Z","timestamp":1750200100000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3377811.3381749"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,6,27]]},"references-count":31,"alternative-id":["10.1145\/3377811.3381749","10.1145\/3377811"],"URL":"https:\/\/doi.org\/10.1145\/3377811.3381749","relation":{},"subject":[],"published":{"date-parts":[[2020,6,27]]},"assertion":[{"value":"2020-10-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}