@inproceedings{806e704e63374650870fcf71a37707ec,
title = "An empirical analysis of flaky tests",
abstract = "Regression testing is a crucial part of software development. It checks that software changes do not break existing functionality. An important assumption of regression testing is that test outcomes are deterministic: an unmodified test is expected to either always pass or always fail for the same code under test. Unfortunately, in practice, some tests-often called flaky tests-have non-deterministic outcomes. Such tests undermine the regression testing as they make it difficult to rely on test results. We present the first extensive study of flaky tests. We study in detail a total of 201 commits that likely fix flaky tests in 51 open-source projects. We classify the most common root causes of flaky tests, identify approaches that could manifest flaky behavior, and describe common strategies that developers use to fix flaky tests. We believe that our insights and implications can help guide future research on the important topic of (avoiding) flaky tests.",
keywords = "Empirical study, Flaky tests, Non-determinism",
author = "Qingzhou Luo and Farah Hariri and Lamyaa Eloussi and Darko Marinov",
note = "Funding Information: We thank John Micco for sharing personal experience about flaky tests at Google, and Sebastian Elbaum and Sai Zhang for the valuable discussions about this work. This research was partially supported by the NSF Grant Nos. CNS-0958199 and CCF-1012759, and the DARPA grant FA8750-12-C-0284. Farah Hariri was also supported by the Saburo Muroga Endowed Fellowship.; 22nd ACM SIGSOFT International Symposium on the Foundations of Software Engineering, FSE 2014 ; Conference date: 16-11-2014 Through 21-11-2014",
year = "2014",
month = nov,
day = "16",
doi = "10.1145/2635868.2635920",
language = "English (US)",
series = "Proceedings of the ACM SIGSOFT Symposium on the Foundations of Software Engineering",
publisher = "Association for Computing Machinery",
pages = "643--653",
booktitle = "22nd ACM SIGSOFT International Symposium on the Foundations of Software Engineering, FSE 2014 - Proceedings",
address = "United States",
}