{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,17]],"date-time":"2026-07-17T14:59:08Z","timestamp":1784300348978,"version":"3.55.0"},"publisher-location":"Cham","reference-count":49,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030853464","type":"print"},{"value":"9783030853471","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-85347-1_3","type":"book-chapter","created":{"date-parts":[[2021,8,27]],"date-time":"2021-08-27T09:06:54Z","timestamp":1630055214000},"page":"29-39","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["What We Talk About When We Talk About Software Test Flakiness"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1281-4058","authenticated-orcid":false,"given":"Morena","family":"Barboni","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8749-1356","authenticated-orcid":false,"given":"Antonia","family":"Bertolino","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1076-0076","authenticated-orcid":false,"given":"Guglielmo","family":"De Angelis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,8,25]]},"reference":[{"key":"3_CR1","unstructured":"Ahmad, A., Leifler, O., Sandahl, K.: Empirical analysis of factors and their effect on test flakiness-practitioners\u2019 perceptions. arXiv preprint arXiv:1906.00673 (2019)"},{"key":"3_CR2","doi-asserted-by":"crossref","unstructured":"Alshammari, A., Morris, C., Hilton, M., Bell, J.: FlakeFlagger: predicting flakiness without rerunning tests. In: Proceedings of ICSE Art. Ev. Track. IEEE (2021)","DOI":"10.1109\/ICSE43902.2021.00140"},{"issue":"1","key":"3_CR3","doi-asserted-by":"publisher","first-page":"19","DOI":"10.1080\/1364557032000119616","volume":"8","author":"H Arksey","year":"2005","unstructured":"Arksey, H., O\u2019Malley, L.: Scoping studies: towards a methodological framework. Int. J. Soc. Res. Methodol. 8(1), 19\u201332 (2005)","journal-title":"Int. J. Soc. Res. Methodol."},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Barboni, M., Bertolino, A., De Angelis, G.: Supplemental material: what we talk about when we talk about software test flakiness (2021)","DOI":"10.1007\/978-3-030-85347-1_3"},{"key":"3_CR5","unstructured":"Bell, J., Legunsen, O., Hilton, M., Eloussi, L., Yung, T., Marinov, D.: DeFlaker: automatically detecting flaky tests. In: Proceedings of ICSE, pp. 433\u2013444. ACM (2018)"},{"issue":"2","key":"3_CR6","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1109\/52.73751","volume":"8","author":"RH Carver","year":"1991","unstructured":"Carver, R.H., Tai, K.C.: Replay and testing for concurrent programs. IEEE Softw. 8(2), 66\u201374 (1991)","journal-title":"IEEE Softw."},{"key":"3_CR7","unstructured":"Champier, C.: Flaky tests caused by a production bug: fix the flakiness, not the bug, February 2019. medium.com"},{"key":"3_CR8","doi-asserted-by":"crossref","unstructured":"Cotroneo, D., Grottke, M., Natella, R., Pietrantuono, R., Trivedi, K.S.: Fault triggers in open-source software: an experience report. In: Proceedings of ISSRE, pp. 178\u2013187. IEEE (2013)","DOI":"10.1109\/ISSRE.2013.6698917"},{"key":"3_CR9","doi-asserted-by":"crossref","unstructured":"Dutta, S., Shi, A., Choudhary, R., Zhang, Z., Jain, A., Misailovic, S.: Detecting flaky tests in probabilistic and machine learning applications. In: Proceedings of ISSTA, pp. 211\u2013224. ACM (2020)","DOI":"10.1145\/3395363.3397366"},{"key":"3_CR10","doi-asserted-by":"crossref","unstructured":"Eck, M., Palomba, F., Castelluccio, M., Bacchelli, A.: Understanding flaky tests: the developer\u2019s perspective. In: Proceedings of ESEC\/FSE, pp. 830\u2013840. ACM (2019)","DOI":"10.1145\/3338906.3338945"},{"key":"3_CR11","unstructured":"Eloussi, L.: Flaky tests (and how to avoid them), September 2016. medium.com"},{"key":"3_CR12","unstructured":"Fowler, M.: Eradicating non-determinism in tests, April 2011"},{"key":"3_CR13","doi-asserted-by":"crossref","unstructured":"Groce, A., Holmes, J.: Practical automatic lightweight nondeterminism and flaky test detection and debugging for Python. In: Proceedings of QRS, pp. 188\u2013195. IEEE (2020)","DOI":"10.1109\/QRS51102.2020.00035"},{"key":"3_CR14","doi-asserted-by":"crossref","unstructured":"Gyori, A., Shi, A., Hariri, F., Marinov, D.: Reliable testing: detecting state-polluting tests to prevent test dependency. In: Proceedings of ISSTA, pp. 223\u2013233. ACM (2015)","DOI":"10.1145\/2771783.2771793"},{"key":"3_CR15","doi-asserted-by":"crossref","unstructured":"King, T.M., Santiago, D., Phillips, J., Clarke, P.J.: Towards a Bayesian network model for predicting flaky automated tests. In: Proceedings of QRS-C, pp. 100\u2013107. IEEE (2018)","DOI":"10.1109\/QRS-C.2018.00031"},{"issue":"2004","key":"3_CR16","first-page":"1","volume":"33","author":"B Kitchenham","year":"2004","unstructured":"Kitchenham, B.: Procedures for performing systematic reviews. Keele UK Keele Univ. 33(2004), 1\u201326 (2004)","journal-title":"Keele UK Keele Univ."},{"key":"3_CR17","doi-asserted-by":"crossref","unstructured":"Kowalczyk, E., Nair, K., Gao, Z., Silberstein, L., Long, T., Memon, A.: Modeling and ranking flaky tests at Apple. In: Proceedings of ICSE-SEIP, pp. 110\u2013119. ACM (2020)","DOI":"10.1145\/3377813.3381370"},{"key":"3_CR18","doi-asserted-by":"crossref","unstructured":"Lam, W., Godefroid, P., Nath, S., Santhiar, A., Thummalapenta, S.: Root causing flaky tests in a large-scale industrial setting. In: Proceedings of ISSTA, pp. 101\u2013111. ACM (2019)","DOI":"10.1145\/3293882.3330570"},{"key":"3_CR19","doi-asserted-by":"crossref","unstructured":"Lam, W., Mu\u015flu, K., Sajnani, H., Thummalapenta, S.: A study on the lifecycle of flaky tests. In: Proceedings of ICSE, pp. 1471\u20131482. ACM (2020)","DOI":"10.1145\/3377811.3381749"},{"key":"3_CR20","doi-asserted-by":"crossref","unstructured":"Lam, W., Oei, R., Shi, A., Marinov, D., Xie, T.: iDFlakies: a framework for detecting and partially classifying flaky tests. In: Proceedings of ICST, pp. 312\u2013322. IEEE (2019)","DOI":"10.1109\/ICST.2019.00038"},{"key":"3_CR21","doi-asserted-by":"crossref","unstructured":"Lam, W., Shi, A., Oei, R., Zhang, S., Ernst, M.D., Xie, T.: Dependent-test-aware regression testing techniques. In: Proceedings of ISSTA, pp. 298\u2013311. ACM (2020)","DOI":"10.1145\/3395363.3397364"},{"key":"3_CR22","doi-asserted-by":"crossref","unstructured":"Lam, W., Winter, S., Astorga, A., Stodden, V., Marinov, D.: Understanding reproducibility and characteristics of flaky tests through test reruns in Java projects. In: Proceedings of ISSRE, pp. 403\u2013413. IEEE (2020)","DOI":"10.1109\/ISSRE5003.2020.00045"},{"issue":"OOPSLA","key":"3_CR23","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3428270","volume":"4","author":"W Lam","year":"2020","unstructured":"Lam, W., Winter, S., Wei, A., Xie, T., Marinov, D., Bell, J.: A large-scale longitudinal study of flaky tests. Proc. ACM Program. Lang. 4(OOPSLA), 1\u201329 (2020)","journal-title":"Proc. ACM Program. Lang."},{"key":"3_CR24","unstructured":"Lee, B.: We have a flaky test problem, November 2019. medium.com"},{"key":"3_CR25","unstructured":"Liviu, S.: A machine learning solution for detecting and mitigating flaky tests, October 2019. medium.com"},{"key":"3_CR26","doi-asserted-by":"crossref","unstructured":"Luo, Q., Hariri, F., Eloussi, L., Marinov, D.: An empirical analysis of flaky tests. In: Proceedings of FSE, pp. 643\u2013653. ACM (2014)","DOI":"10.1145\/2635868.2635920"},{"key":"3_CR27","doi-asserted-by":"crossref","unstructured":"Machalica, M., Samylkin, A., Porth, M., Chandra, S.: Predictive test selection. In: Proceedings of ICSE-SEIP, pp. 91\u2013100. IEEE (2019)","DOI":"10.1109\/ICSE-SEIP.2019.00018"},{"key":"3_CR28","doi-asserted-by":"crossref","unstructured":"Malm, J., Causevic, A., Lisper, B., Eldh, S.: Automated analysis of flakiness-mitigating delays. In: Proceedings of AST, pp. 81\u201384. IEEE (2020)","DOI":"10.1145\/3387903.3389320"},{"key":"3_CR29","unstructured":"Micco, J.: Flaky tests at Google and how we mitigate them, May 2016D"},{"issue":"1","key":"3_CR30","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s12874-017-0458-6","volume":"18","author":"Z Munn","year":"2018","unstructured":"Munn, Z., Peters, M.D., Stern, C., Tufanaru, C., McArthur, A., Aromataris, E.: Systematic review or scoping review? Guidance for authors when choosing between a systematic or scoping review approach. BMC Med. Res. Methodol. 18(1), 1\u20137 (2018)","journal-title":"BMC Med. Res. Methodol."},{"key":"3_CR31","unstructured":"Otrebski, K.: Flaky tests, April 2018. medium.com"},{"key":"3_CR32","unstructured":"Palmer, J.: Test flakiness - methods for identifying and dealing with flaky tests, November 2019. medium.com"},{"key":"3_CR33","doi-asserted-by":"crossref","unstructured":"Parry, O., Kapfhammer, G.M., Hilton, M., McMinn, P.: Flake it\u2019till you make it: using automated repair to induce and fix latent test flakiness. In: Proceedings of ICSE Workshops, pp. 11\u201312. ACM (2020)","DOI":"10.1145\/3387940.3392177"},{"key":"3_CR34","doi-asserted-by":"crossref","unstructured":"Presler-Marshall, K., Horton, E., Heckman, S., Stolee, K.: Wait, wait. No, tell me. Analyzing selenium configuration effects on test flakiness. In: Proceedings of Wksp AST, pp. 7\u201313. IEEE (2019)","DOI":"10.1109\/AST.2019.000-1"},{"key":"3_CR35","doi-asserted-by":"crossref","unstructured":"Rahman, M.T., Rigby, P.C.: The impact of failing, flaky, and high failure tests on the number of crash reports associated with Firefox builds. In: Proceedings of ESEC\/FSE, pp. 857\u2013862. ACM (2018)","DOI":"10.1145\/3236024.3275529"},{"key":"3_CR36","doi-asserted-by":"crossref","unstructured":"Shi, A., Bell, J., Marinov, D.: Mitigating the effects of flaky tests on mutation testing. In: Proceedings of ISSTA, pp. 112\u2013122. ACM (2019)","DOI":"10.1145\/3293882.3330568"},{"key":"3_CR37","doi-asserted-by":"crossref","unstructured":"Shi, A., Gyori, A., Legunsen, O., Marinov, D.: Detecting assumptions on deterministic implementations of non-deterministic specifications. In: Proceedings of ICST, pp. 80\u201390. IEEE (2016)","DOI":"10.1109\/ICST.2016.40"},{"key":"3_CR38","doi-asserted-by":"crossref","unstructured":"Shi, A., Lam, W., Oei, R., Xie, T., Marinov, D.: iFixFlakies: a framework for automatically fixing order-dependent flaky tests. In: Proceedings of ESEC\/FSE, pp. 545\u2013555. ACM (2019)","DOI":"10.1145\/3338906.3338925"},{"key":"3_CR39","doi-asserted-by":"crossref","unstructured":"Silva, D., Teixeira, L., d\u2019Amorim, M.: Shake it! Detecting flaky tests caused by concurrency with Shaker. In: Proceedings of ICSME, pp. 301\u2013311. IEEE (2020)","DOI":"10.1109\/ICSME46990.2020.00037"},{"key":"3_CR40","unstructured":"S\u0142api\u0144ski, M.: What is flakiness and how we deal with it, February 2020. medium.com"},{"key":"3_CR41","unstructured":"Stosik, D.: Dealing with flaky tests, November 2019. medium.com"},{"key":"3_CR42","doi-asserted-by":"crossref","unstructured":"Strandberg, P.E., Ostrand, T.J., Weyuker, E.J., Afzal, W., Sundmark, D.: Intermittently failing tests in the embedded systems domain. In: Proceedings of ISSTA, pp. 337\u2013348. ACM (2020)","DOI":"10.1145\/3395363.3397359"},{"key":"3_CR43","doi-asserted-by":"crossref","unstructured":"Terragni, V., Salza, P., Ferrucci, F.: A container-based infrastructure for fuzzy-driven root causing of flaky tests. In: Proceedings of ICSE-NIER, pp. 69\u201372. IEEE (2020)","DOI":"10.1145\/3377816.3381742"},{"key":"3_CR44","doi-asserted-by":"crossref","unstructured":"Thorve, S., Sreshtha, C., Meng, N.: An empirical study of flaky tests in android apps. In: Proceedings of ICSME, pp. 534\u2013538. IEEE (2018)","DOI":"10.1109\/ICSME.2018.00062"},{"key":"3_CR45","doi-asserted-by":"crossref","unstructured":"Vahabzadeh, A., Fard, A.M., Mesbah, A.: An empirical study of bugs in test code. In: Proceedings of ICSME, pp. 101\u2013110. IEEE (2015)","DOI":"10.1109\/ICSM.2015.7332456"},{"key":"3_CR46","doi-asserted-by":"crossref","unstructured":"Waterloo, M., Person, S., Elbaum, S.: Test analysis: searching for faults in tests (N). In: Proceedings of ASE. IEEE, November 2015","DOI":"10.1109\/ASE.2015.37"},{"key":"3_CR47","doi-asserted-by":"crossref","unstructured":"Zhang, S., et al.: Empirically revisiting the test independence assumption. In: Proceedings of ISSTA, pp. 385\u2013396. ACM (2014)","DOI":"10.1145\/2610384.2610404"},{"key":"3_CR48","doi-asserted-by":"crossref","unstructured":"Ziftci, C., Cavalcanti, D.: De-Flake your tests: automatically locating root causes of flaky tests in code at Google. In: Proceedings of ICSME, pp. 736\u2013745. IEEE (2020)","DOI":"10.1109\/ICSME46990.2020.00083"},{"key":"3_CR49","doi-asserted-by":"crossref","unstructured":"Zolfaghari, B., Parizi, R.M., Srivastava, G., Hailemariam, Y.: Root causing, detecting, and fixing flaky tests: state of the art and future roadmap. Softw.: Pract. Exp. 51, 851\u2013867 (2020)","DOI":"10.1002\/spe.2929"}],"container-title":["Communications in Computer and Information Science","Quality of Information and Communications Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-85347-1_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,8]],"date-time":"2023-01-08T05:54:06Z","timestamp":1673157246000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-85347-1_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030853464","9783030853471"],"references-count":49,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-85347-1_3","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"value":"1865-0929","type":"print"},{"value":"1865-0937","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"25 August 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"QUATIC","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on the Quality of Information and Communications Technology","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Algarve","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Portugal","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"8 September 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 September 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"quatic2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2021.quatic.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"98","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"30","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"31% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}